From 3ef83057161e6d973f79805340d4c3d210425465 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Sun, 30 May 2021 16:08:03 +0200 Subject: cleanup: old preliminary tasks --- .../prometheus/exporter/base/tasks/main.yml | 21 ++++++++++----------- .../exporter/base/templates/nginx-vhost.conf.j2 | 15 --------------- 2 files changed, 10 insertions(+), 26 deletions(-) delete mode 100644 roles/monitoring/prometheus/exporter/base/templates/nginx-vhost.conf.j2 (limited to 'roles/monitoring/prometheus/exporter/base') diff --git a/roles/monitoring/prometheus/exporter/base/tasks/main.yml b/roles/monitoring/prometheus/exporter/base/tasks/main.yml index c3a04bd9..7982f1f9 100644 --- a/roles/monitoring/prometheus/exporter/base/tasks/main.yml +++ b/roles/monitoring/prometheus/exporter/base/tasks/main.yml @@ -1,16 +1,15 @@ --- - name: create main configuration directories - loop: - - exporters-available - - exporters-enabled file: - path: "/etc/prometheus-exporter/{{ item }}" + path: "/etc/prometheus/exporters" state: directory -- name: install nginx vhost - vars: - nginx_vhost: - name: prometheus-exporter - content: "{{ lookup('template', 'nginx-vhost.conf.j2') }}" - include_role: - name: nginx/vhost +- name: install apt packages + apt: + name: prom-exporter-exporter + state: present + +## TODO: +## - systemd service unit +## - add snippet to exporter-exporter config-dir +## - create certificate/key diff --git a/roles/monitoring/prometheus/exporter/base/templates/nginx-vhost.conf.j2 b/roles/monitoring/prometheus/exporter/base/templates/nginx-vhost.conf.j2 deleted file mode 100644 index e032ca3d..00000000 --- a/roles/monitoring/prometheus/exporter/base/templates/nginx-vhost.conf.j2 +++ /dev/null @@ -1,15 +0,0 @@ -server { - listen {{ prometheus_exporter_port }}; - listen [::]:{{ prometheus_exporter_port }}; - server_name _; - - ## TODO: configure ssl - - location / { - return 404 "unknown exporter: $uri\n"; - } - include /etc/prometheus-exporter/exporters-enabled/*; - - access_log /var/log/nginx/access-prometheus-exporter.log; - error_log /var/log/nginx/error-prometheus-exporter.log; -} -- cgit v1.2.3 From acfdc3ae8545177547fa75510cb9e56e0b909156 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Sun, 30 May 2021 23:24:32 +0200 Subject: prometheus basic exporter base role --- chaos-at-home/ch-mon.yml | 1 + .../prometheus/exporter/base/defaults/main.yml | 2 +- .../prometheus/exporter/base/handlers/main.yml | 5 +++ .../prometheus/exporter/base/tasks/main.yml | 37 ++++++++++++++++++---- .../prometheus/exporter/base/templates/service.j2 | 31 ++++++++++++++++++ 5 files changed, 69 insertions(+), 7 deletions(-) create mode 100644 roles/monitoring/prometheus/exporter/base/handlers/main.yml create mode 100644 roles/monitoring/prometheus/exporter/base/templates/service.j2 (limited to 'roles/monitoring/prometheus/exporter/base') diff --git a/chaos-at-home/ch-mon.yml b/chaos-at-home/ch-mon.yml index fb0eff53..b069bbf8 100644 --- a/chaos-at-home/ch-mon.yml +++ b/chaos-at-home/ch-mon.yml @@ -10,3 +10,4 @@ - role: storage/lvm/groups - role: apt-repo/spreadspace - role: monitoring/prometheus/server + - role: monitoring/prometheus/exporter/base diff --git a/roles/monitoring/prometheus/exporter/base/defaults/main.yml b/roles/monitoring/prometheus/exporter/base/defaults/main.yml index 5f8ce103..963763a5 100644 --- a/roles/monitoring/prometheus/exporter/base/defaults/main.yml +++ b/roles/monitoring/prometheus/exporter/base/defaults/main.yml @@ -1,2 +1,2 @@ --- -prometheus_exporter_port: 9000 +prometheus_exporter_listen: ":9999" diff --git a/roles/monitoring/prometheus/exporter/base/handlers/main.yml b/roles/monitoring/prometheus/exporter/base/handlers/main.yml new file mode 100644 index 00000000..ebd760cf --- /dev/null +++ b/roles/monitoring/prometheus/exporter/base/handlers/main.yml @@ -0,0 +1,5 @@ +--- +- name: restart prometheus-exporter-exporter + service: + name: prometheus-exporter-exporter + state: restarted diff --git a/roles/monitoring/prometheus/exporter/base/tasks/main.yml b/roles/monitoring/prometheus/exporter/base/tasks/main.yml index 7982f1f9..fab6ff7b 100644 --- a/roles/monitoring/prometheus/exporter/base/tasks/main.yml +++ b/roles/monitoring/prometheus/exporter/base/tasks/main.yml @@ -1,15 +1,40 @@ --- -- name: create main configuration directories - file: - path: "/etc/prometheus/exporters" - state: directory +- name: check if prometheus apt component of spreadspace repo is enabled + assert: + msg: "please enable the 'prometheus' component of spreadspace repo using 'spreadspace_apt_repo_components'" + that: + - spreadspace_apt_repo_components is defined + - "'prometheus' in spreadspace_apt_repo_components" - name: install apt packages apt: name: prom-exporter-exporter state: present +- name: create configuration directories + file: + path: /etc/prometheus/exporter/enabled + state: directory + +- name: add user for prometheus-exporter + user: + name: prometheus-exporter + system: yes + home: /nonexistent + create_home: no + ## TODO: -## - systemd service unit -## - add snippet to exporter-exporter config-dir ## - create certificate/key + +- name: generate systemd service unit + template: + src: service.j2 + dest: /etc/systemd/system/prometheus-exporter-exporter.service + notify: restart prometheus-exporter-exporter + +- name: make sure prometheus-exporter-exporter is enabled and started + systemd: + name: prometheus-exporter-exporter.service + daemon_reload: yes + state: started + enabled: yes diff --git a/roles/monitoring/prometheus/exporter/base/templates/service.j2 b/roles/monitoring/prometheus/exporter/base/templates/service.j2 new file mode 100644 index 00000000..6069fc79 --- /dev/null +++ b/roles/monitoring/prometheus/exporter/base/templates/service.j2 @@ -0,0 +1,31 @@ +[Unit] +Description=Prometheus exporter proxy + +[Service] +Restart=always +User=prometheus-exporter +ExecStart=/usr/bin/prometheus-exporter-exporter --config.dirs=/etc/prometheus/exporter/enabled --config.file="" --web.listen-address="{{ prometheus_exporter_listen }}" + +# systemd hardening-options +AmbientCapabilities= +CapabilityBoundingSet= +DeviceAllow=/dev/null rw +DevicePolicy=strict +LockPersonality=true +MemoryDenyWriteExecute=true +NoNewPrivileges=true +PrivateDevices=true +PrivateTmp=true +PrivateUsers=true +ProtectControlGroups=true +ProtectHome=true +ProtectKernelModules=true +ProtectKernelTunables=true +ProtectSystem=strict +RemoveIPC=true +RestrictNamespaces=true +RestrictRealtime=true +SystemCallArchitectures=native + +[Install] +WantedBy=multi-user.target -- cgit v1.2.3 From 35fb88969a6cb85d8ba7541820acf3b0ff891055 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Mon, 31 May 2021 22:41:50 +0200 Subject: prometheus: initial node exporter role --- chaos-at-home/ch-mon.yml | 1 + .../prometheus/exporter/base/templates/service.j2 | 1 + .../prometheus/exporter/node/defaults/main.yml | 5 +++ .../prometheus/exporter/node/handlers/main.yml | 6 ++++ .../prometheus/exporter/node/tasks/main.yml | 36 ++++++++++++++++++++-- .../prometheus/exporter/node/templates/service.j2 | 10 ++++++ 6 files changed, 56 insertions(+), 3 deletions(-) create mode 100644 roles/monitoring/prometheus/exporter/node/defaults/main.yml create mode 100644 roles/monitoring/prometheus/exporter/node/templates/service.j2 (limited to 'roles/monitoring/prometheus/exporter/base') diff --git a/chaos-at-home/ch-mon.yml b/chaos-at-home/ch-mon.yml index b069bbf8..2cb69484 100644 --- a/chaos-at-home/ch-mon.yml +++ b/chaos-at-home/ch-mon.yml @@ -11,3 +11,4 @@ - role: apt-repo/spreadspace - role: monitoring/prometheus/server - role: monitoring/prometheus/exporter/base + - role: monitoring/prometheus/exporter/node diff --git a/roles/monitoring/prometheus/exporter/base/templates/service.j2 b/roles/monitoring/prometheus/exporter/base/templates/service.j2 index 6069fc79..e2c54d6c 100644 --- a/roles/monitoring/prometheus/exporter/base/templates/service.j2 +++ b/roles/monitoring/prometheus/exporter/base/templates/service.j2 @@ -5,6 +5,7 @@ Description=Prometheus exporter proxy Restart=always User=prometheus-exporter ExecStart=/usr/bin/prometheus-exporter-exporter --config.dirs=/etc/prometheus/exporter/enabled --config.file="" --web.listen-address="{{ prometheus_exporter_listen }}" +{# TODO: implement reloading once the exporter_exporter supports this #} # systemd hardening-options AmbientCapabilities= diff --git a/roles/monitoring/prometheus/exporter/node/defaults/main.yml b/roles/monitoring/prometheus/exporter/node/defaults/main.yml new file mode 100644 index 00000000..5eff7844 --- /dev/null +++ b/roles/monitoring/prometheus/exporter/node/defaults/main.yml @@ -0,0 +1,5 @@ +--- +prometheus_exporter_node_disable_collectors: [] + +prometheus_exporter_node_extra_collectors: +- ntp diff --git a/roles/monitoring/prometheus/exporter/node/handlers/main.yml b/roles/monitoring/prometheus/exporter/node/handlers/main.yml index 9c62baf6..3e1b2000 100644 --- a/roles/monitoring/prometheus/exporter/node/handlers/main.yml +++ b/roles/monitoring/prometheus/exporter/node/handlers/main.yml @@ -3,3 +3,9 @@ service: name: prometheus-node-exporter state: restarted + +- name: reload prometheus-exporter-exporter + service: + name: prometheus-exporter-exporter + ## TODO: implement reload once exporter_exporter supports this... + state: restarted diff --git a/roles/monitoring/prometheus/exporter/node/tasks/main.yml b/roles/monitoring/prometheus/exporter/node/tasks/main.yml index 0758eb3f..694dafb0 100644 --- a/roles/monitoring/prometheus/exporter/node/tasks/main.yml +++ b/roles/monitoring/prometheus/exporter/node/tasks/main.yml @@ -1,9 +1,39 @@ --- +- name: check if prometheus apt component of spreadspace repo is enabled + assert: + msg: "please enable the 'prometheus' component of spreadspace repo using 'spreadspace_apt_repo_components'" + that: + - spreadspace_apt_repo_components is defined + - "'prometheus' in spreadspace_apt_repo_components" + - name: install apt packages apt: name: prom-exporter-node state: present -## TODO: -## - systemd service unit -## - add snippet to exporter-exporter config-dir +- name: create directory for textfile collector + file: + path: /var/lib/prometheus-node-exporter/textfile-collector + state: directory + +- name: generate systemd service unit + template: + src: service.j2 + dest: /etc/systemd/system/prometheus-node-exporter.service + notify: restart prometheus-node-exporter + +- name: make sure prometheus-exporter-exporter is enabled and started + systemd: + name: prometheus-node-exporter.service + daemon_reload: yes + state: started + enabled: yes + +- name: register exporter + copy: + content: | + method: http + http: + port: 9100 + dest: /etc/prometheus/exporter/enabled/node.yml + notify: reload prometheus-exporter-exporter diff --git a/roles/monitoring/prometheus/exporter/node/templates/service.j2 b/roles/monitoring/prometheus/exporter/node/templates/service.j2 new file mode 100644 index 00000000..c3b46472 --- /dev/null +++ b/roles/monitoring/prometheus/exporter/node/templates/service.j2 @@ -0,0 +1,10 @@ +[Unit] +Description=Prometheus exporter + +[Service] +Restart=always +User=prometheus-exporter +ExecStart=/usr/bin/prometheus-node-exporter --web.listen-address="127.0.0.1:9100" --web.disable-exporter-metrics --collector.textfile.directory="/var/lib/prometheus-node-exporter/textfile-collector" {% for collector in prometheus_exporter_node_disable_collectors %} --no-collector.{{ collector }}{% endfor %}{% for collector in prometheus_exporter_node_extra_collectors %} --collector.{{ collector }}{% endfor %}{{ '' }} + +[Install] +WantedBy=multi-user.target -- cgit v1.2.3 From 96232fb34906f9efd98189838141d896668d3dd8 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Wed, 2 Jun 2021 02:25:33 +0200 Subject: prometheus: preliminary tls support for exporter connections --- .../prometheus/exporter/base/tasks/main.yml | 4 +- .../prometheus/exporter/base/tasks/tls.yml | 61 ++++++++++++++++++++++ .../prometheus/exporter/base/templates/service.j2 | 2 +- .../prometheus/server/templates/prometheus.yml.j2 | 7 +++ 4 files changed, 71 insertions(+), 3 deletions(-) create mode 100644 roles/monitoring/prometheus/exporter/base/tasks/tls.yml (limited to 'roles/monitoring/prometheus/exporter/base') diff --git a/roles/monitoring/prometheus/exporter/base/tasks/main.yml b/roles/monitoring/prometheus/exporter/base/tasks/main.yml index fab6ff7b..9a214f39 100644 --- a/roles/monitoring/prometheus/exporter/base/tasks/main.yml +++ b/roles/monitoring/prometheus/exporter/base/tasks/main.yml @@ -23,8 +23,8 @@ home: /nonexistent create_home: no -## TODO: -## - create certificate/key +- name: create TLS certificate and key + import_tasks: tls.yml - name: generate systemd service unit template: diff --git a/roles/monitoring/prometheus/exporter/base/tasks/tls.yml b/roles/monitoring/prometheus/exporter/base/tasks/tls.yml new file mode 100644 index 00000000..b2731b09 --- /dev/null +++ b/roles/monitoring/prometheus/exporter/base/tasks/tls.yml @@ -0,0 +1,61 @@ +--- +- name: install python-cryptoraphy + apt: + name: "{{ python_basename }}-cryptography" + state: present + +- name: create base directory + file: + path: /etc/ssl/prometheus + state: directory + +- name: create exporter cert/key directory + file: + path: /etc/ssl/prometheus/exporter + state: directory + owner: root + group: prometheus-exporter + mode: 0750 + +- name: create exporter private key + openssl_privatekey: + path: /etc/ssl/prometheus/exporter/key.pem + type: RSA + size: 4096 + owner: prometheus-exporter + group: prometheus-exporter + mode: 0400 + notify: restart prometheus-exporter-exporter + +- name: create signing request for exporter certificate + openssl_csr: + path: /etc/ssl/prometheus/exporter/csr.pem + privatekey_path: /etc/ssl/prometheus/exporter/key.pem + CN: "{{ inventory_hostname }}" + subject_alt_name: + - "DNS:{{ host_name }}.{{ host_domain }}" + - "IP:{{ ansible_default_ipv4.address }}" + key_usage: + - digitalSignature + key_usage_critical: yes + extended_key_usage: + - serverAuth + extended_key_usage_critical: yes + basic_constraints: + - 'CA:FALSE' + basic_constraints_critical: yes + +## TODO: implement remote singing using server + +- name: create exporter certificate + openssl_certificate: + path: /etc/ssl/prometheus/exporter/crt.pem + csr_path: /etc/ssl/prometheus/exporter/csr.pem + provider: ownca + ownca_path: /etc/ssl/prometheus/ca-crt.pem + ownca_privatekey_path: /etc/ssl/prometheus/ca/key.pem + ownca_digest: sha256 + ownca_not_after: "+18250d" ## 50 years + notify: restart prometheus-exporter-exporter + +## TODO: install /etc/ssl/prometheus/ca-crt.pem from server diff --git a/roles/monitoring/prometheus/exporter/base/templates/service.j2 b/roles/monitoring/prometheus/exporter/base/templates/service.j2 index e2c54d6c..c24baf43 100644 --- a/roles/monitoring/prometheus/exporter/base/templates/service.j2 +++ b/roles/monitoring/prometheus/exporter/base/templates/service.j2 @@ -4,7 +4,7 @@ Description=Prometheus exporter proxy [Service] Restart=always User=prometheus-exporter -ExecStart=/usr/bin/prometheus-exporter-exporter --config.dirs=/etc/prometheus/exporter/enabled --config.file="" --web.listen-address="{{ prometheus_exporter_listen }}" +ExecStart=/usr/bin/prometheus-exporter-exporter -config.dirs=/etc/prometheus/exporter/enabled -config.file="" -web.listen-address="" -web.tls.listen-address="{{ prometheus_exporter_listen }}" -web.tls.cert="/etc/ssl/prometheus/exporter/crt.pem" -web.tls.key="/etc/ssl/prometheus/exporter/key.pem" --web.tls.ca="/etc/ssl/prometheus/ca-crt.pem" -web.tls.verify {# TODO: implement reloading once the exporter_exporter supports this #} # systemd hardening-options diff --git a/roles/monitoring/prometheus/server/templates/prometheus.yml.j2 b/roles/monitoring/prometheus/server/templates/prometheus.yml.j2 index 007afa90..e94ea043 100644 --- a/roles/monitoring/prometheus/server/templates/prometheus.yml.j2 +++ b/roles/monitoring/prometheus/server/templates/prometheus.yml.j2 @@ -9,6 +9,8 @@ scrape_configs: - job_name: 'prometheus' static_configs: - targets: ['localhost:9090'] + labels: + instance: "{{ inventory_hostname }}" {% for job in prometheus_server_jobs %} - job_name: '{{ job }}' @@ -16,6 +18,11 @@ scrape_configs: params: module: - {{ job }} + scheme: https + tls_config: + ca_file: /etc/ssl/prometheus/ca-crt.pem + cert_file: /etc/ssl/prometheus/server/exporter-crt.pem + key_file: /etc/ssl/prometheus/server/exporter-key.pem file_sd_configs: - files: - "/etc/prometheus/jobs/{{ job }}/*.yml" -- cgit v1.2.3 From 6082a92fa86d121d3ea4256859ee4c9d412e78c0 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Thu, 10 Jun 2021 01:15:32 +0200 Subject: promethues: remote certificate signing for exporter/base --- chaos-at-home/ch-testvm-prometheus.yml | 7 +++- inventory/host_vars/ch-testvm-prometheus.yml | 3 ++ roles/monitoring/prometheus/ca/tasks/main.yml | 2 +- .../prometheus/exporter/base/tasks/tls.yml | 49 +++++++++++++++++++--- roles/monitoring/prometheus/server/tasks/tls.yml | 34 ++++++++++----- .../prometheus/server/templates/prometheus.yml.j2 | 16 +++---- 6 files changed, 85 insertions(+), 26 deletions(-) (limited to 'roles/monitoring/prometheus/exporter/base') diff --git a/chaos-at-home/ch-testvm-prometheus.yml b/chaos-at-home/ch-testvm-prometheus.yml index a34d58e3..9caa2f9a 100644 --- a/chaos-at-home/ch-testvm-prometheus.yml +++ b/chaos-at-home/ch-testvm-prometheus.yml @@ -7,5 +7,8 @@ - role: core/sshd/base - role: core/zsh - role: core/ntp - - role: kubernetes/base - - role: kubernetes/standalone/base + - role: apt-repo/spreadspace + - role: monitoring/prometheus/exporter/base + - role: monitoring/prometheus/exporter/node + # - role: kubernetes/base + # - role: kubernetes/standalone/base diff --git a/inventory/host_vars/ch-testvm-prometheus.yml b/inventory/host_vars/ch-testvm-prometheus.yml index d11d565c..e539735f 100644 --- a/inventory/host_vars/ch-testvm-prometheus.yml +++ b/inventory/host_vars/ch-testvm-prometheus.yml @@ -33,6 +33,9 @@ network: - *_network_primary_ +spreadspace_apt_repo_components: + - prometheus + containerd_storage: type: lvm diff --git a/roles/monitoring/prometheus/ca/tasks/main.yml b/roles/monitoring/prometheus/ca/tasks/main.yml index 9f166321..cde4a267 100644 --- a/roles/monitoring/prometheus/ca/tasks/main.yml +++ b/roles/monitoring/prometheus/ca/tasks/main.yml @@ -34,7 +34,6 @@ useCommonNameForSAN: no key_usage: - cRLSign - - digitalSignature - keyCertSign key_usage_critical: yes basic_constraints: @@ -50,3 +49,4 @@ provider: selfsigned selfsigned_digest: sha256 selfsigned_not_after: "+18250d" ## 50 years + selfsigned_create_subject_key_identifier: always_create diff --git a/roles/monitoring/prometheus/exporter/base/tasks/tls.yml b/roles/monitoring/prometheus/exporter/base/tasks/tls.yml index b2731b09..72186acb 100644 --- a/roles/monitoring/prometheus/exporter/base/tasks/tls.yml +++ b/roles/monitoring/prometheus/exporter/base/tasks/tls.yml @@ -45,17 +45,56 @@ - 'CA:FALSE' basic_constraints_critical: yes -## TODO: implement remote singing using server +- name: slurp CSR + slurp: + src: /etc/ssl/prometheus/exporter/csr.pem + register: prometheus_exporter_server_csr -- name: create exporter certificate - openssl_certificate: +- name: check if exporter certificate exists + stat: path: /etc/ssl/prometheus/exporter/crt.pem - csr_path: /etc/ssl/prometheus/exporter/csr.pem + register: prometheus_exporter_server_cert + +- name: read exporter client certificate issuer key id and validity + when: prometheus_exporter_server_cert.stat.exists + openssl_certificate_info: + path: /etc/ssl/prometheus/exporter/crt.pem + valid_at: + ten_years: '+3650d' + register: prometheus_exporter_server_cert_info + +- name: slurp existing exporter certificate + when: prometheus_exporter_server_cert.stat.exists + slurp: + src: /etc/ssl/prometheus/exporter/crt.pem + register: prometheus_exporter_server_cert_current + +- name: generate exporter certificate + delegate_to: "{{ promethues_server }}" + community.crypto.x509_certificate_pipe: + content: "{{ prometheus_exporter_server_cert_current.content | default('') | b64decode }}" + csr_content: "{{ prometheus_exporter_server_csr.content | b64decode }}" provider: ownca ownca_path: /etc/ssl/prometheus/ca-crt.pem ownca_privatekey_path: /etc/ssl/prometheus/ca/key.pem ownca_digest: sha256 ownca_not_after: "+18250d" ## 50 years + force: "{{ prometheus_exporter_server_cert.stat.exists and (not prometheus_exporter_server_cert_info.valid_at.ten_years) }}" + register: prometheus_exporter_server_cert + +- name: store exporter certificate + copy: + content: "{{ prometheus_exporter_server_cert.certificate }}" + dest: /etc/ssl/prometheus/exporter/crt.pem notify: restart prometheus-exporter-exporter -## TODO: install /etc/ssl/prometheus/ca-crt.pem from server +- name: slurp CA certificate + delegate_to: "{{ promethues_server }}" + slurp: + src: /etc/ssl/prometheus/ca-crt.pem + register: prometheus_exporter_ca_certificate + +- name: install CA certificate + copy: + content: "{{ prometheus_exporter_ca_certificate.content | b64decode }}" + dest: /etc/ssl/prometheus/ca-crt.pem diff --git a/roles/monitoring/prometheus/server/tasks/tls.yml b/roles/monitoring/prometheus/server/tasks/tls.yml index 5c112e12..940c69b1 100644 --- a/roles/monitoring/prometheus/server/tasks/tls.yml +++ b/roles/monitoring/prometheus/server/tasks/tls.yml @@ -17,9 +17,9 @@ group: prometheus mode: 0750 -- name: create private key to connect to exporter +- name: create private key for scrape-client certificate openssl_privatekey: - path: /etc/ssl/prometheus/server/exporter-key.pem + path: /etc/ssl/prometheus/server/scrape-key.pem type: RSA size: 4096 owner: prometheus @@ -27,10 +27,10 @@ mode: 0400 notify: reload prometheus -- name: create signing request for client certificate to connect to exporter +- name: create signing request for scrape-client certificate openssl_csr: - path: /etc/ssl/prometheus/server/exporter-csr.pem - privatekey_path: /etc/ssl/prometheus/server/exporter-key.pem + path: /etc/ssl/prometheus/server/scrape-csr.pem + privatekey_path: /etc/ssl/prometheus/server/scrape-key.pem CN: "{{ inventory_hostname }}" subject_alt_name: - "DNS:{{ host_name }}.{{ host_domain }}" @@ -45,17 +45,31 @@ - 'CA:FALSE' basic_constraints_critical: yes +## TODO: install /etc/ssl/prometheus/ca-crt.pem from CA host + +- name: check if scrape-client certificate exists + stat: + path: /etc/ssl/prometheus/server/scrape-crt.pem + register: prometheus_server_scrape_client_cert + +- name: check scrape-client certificate validity + when: prometheus_server_scrape_client_cert.stat.exists + openssl_certificate_info: + path: /etc/ssl/prometheus/server/scrape-crt.pem + valid_at: + ten_years: '+3650d' + register: prometheus_server_scrape_client_cert_info + ## TODO: implement remote signing? -- name: create client certificate to connect to exporter +- name: create scrape-client certificate openssl_certificate: - path: /etc/ssl/prometheus/server/exporter-crt.pem - csr_path: /etc/ssl/prometheus/server/exporter-csr.pem + path: /etc/ssl/prometheus/server/scrape-crt.pem + csr_path: /etc/ssl/prometheus/server/scrape-csr.pem provider: ownca ownca_path: /etc/ssl/prometheus/ca-crt.pem ownca_privatekey_path: /etc/ssl/prometheus/ca/key.pem ownca_digest: sha256 ownca_not_after: "+18250d" ## 50 years + force: "{{ prometheus_server_scrape_client_cert.stat.exists and (not prometheus_server_scrape_client_cert_info.valid_at.ten_years) }}" notify: reload prometheus - -## TODO: install /etc/ssl/prometheus/ca-crt.pem from server diff --git a/roles/monitoring/prometheus/server/templates/prometheus.yml.j2 b/roles/monitoring/prometheus/server/templates/prometheus.yml.j2 index 5eb7c570..3975c74d 100644 --- a/roles/monitoring/prometheus/server/templates/prometheus.yml.j2 +++ b/roles/monitoring/prometheus/server/templates/prometheus.yml.j2 @@ -23,8 +23,8 @@ scrape_configs: scheme: https tls_config: ca_file: /etc/ssl/prometheus/ca-crt.pem - cert_file: /etc/ssl/prometheus/server/exporter-crt.pem - key_file: /etc/ssl/prometheus/server/exporter-key.pem + cert_file: /etc/ssl/prometheus/server/scrape-crt.pem + key_file: /etc/ssl/prometheus/server/scrape-key.pem file_sd_configs: - files: - "/etc/prometheus/jobs/{{ job }}/*.yml" @@ -40,8 +40,8 @@ scrape_configs: scheme: https tls_config: ca_file: /etc/ssl/prometheus/ca-crt.pem - cert_file: /etc/ssl/prometheus/server/exporter-crt.pem - key_file: /etc/ssl/prometheus/server/exporter-key.pem + cert_file: /etc/ssl/prometheus/server/scrape-crt.pem + key_file: /etc/ssl/prometheus/server/scrape-key.pem static_configs: - targets: - 62.99.185.129 @@ -63,8 +63,8 @@ scrape_configs: scheme: https tls_config: ca_file: /etc/ssl/prometheus/ca-crt.pem - cert_file: /etc/ssl/prometheus/server/exporter-crt.pem - key_file: /etc/ssl/prometheus/server/exporter-key.pem + cert_file: /etc/ssl/prometheus/server/scrape-crt.pem + key_file: /etc/ssl/prometheus/server/scrape-key.pem static_configs: - targets: - web.chaos-at-home.org @@ -85,8 +85,8 @@ scrape_configs: scheme: https tls_config: ca_file: /etc/ssl/prometheus/ca-crt.pem - cert_file: /etc/ssl/prometheus/server/exporter-crt.pem - key_file: /etc/ssl/prometheus/server/exporter-key.pem + cert_file: /etc/ssl/prometheus/server/scrape-crt.pem + key_file: /etc/ssl/prometheus/server/scrape-key.pem static_configs: - targets: - 192.168.32.230:222 -- cgit v1.2.3 From 1e9d610bb87ce6f0cb1e5a8d44f09616f90273e2 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Fri, 18 Jun 2021 01:24:40 +0200 Subject: prometheus enable/disable targets for jobs --- .../group_vars/promzone-chaos-at-home/vars.yml | 12 ++++++--- roles/monitoring/prometheus/ca/tasks/main.yml | 2 +- .../prometheus/exporter/base/tasks/tls.yml | 4 +-- .../prometheus/server/filter_plugins/prometheus.py | 29 ++++++++++++++++++++++ roles/monitoring/prometheus/server/tasks/main.yml | 11 ++++++-- 5 files changed, 49 insertions(+), 9 deletions(-) create mode 100644 roles/monitoring/prometheus/server/filter_plugins/prometheus.py (limited to 'roles/monitoring/prometheus/exporter/base') diff --git a/inventory/group_vars/promzone-chaos-at-home/vars.yml b/inventory/group_vars/promzone-chaos-at-home/vars.yml index 2345292b..078576f1 100644 --- a/inventory/group_vars/promzone-chaos-at-home/vars.yml +++ b/inventory/group_vars/promzone-chaos-at-home/vars.yml @@ -1,9 +1,13 @@ --- -promethues_server: ch-mon -promethues_zone_name: chaos@home - -prometheus_zone_targets: "{{ groups['promzone-chaos-at-home'] }}" +prometheus_scrape_endpoint: "{{ network.primary.address | ipaddr('address') }}:9999" prometheus_exporters_extra: [] prometheus_exporters_default: - node + +prometheus_server: ch-mon +prometheus_server_jobs: + - node + +prometheus_zone_name: chaos@home +prometheus_zone_targets: "{{ groups['promzone-chaos-at-home'] }}" diff --git a/roles/monitoring/prometheus/ca/tasks/main.yml b/roles/monitoring/prometheus/ca/tasks/main.yml index cde4a267..064cb6e8 100644 --- a/roles/monitoring/prometheus/ca/tasks/main.yml +++ b/roles/monitoring/prometheus/ca/tasks/main.yml @@ -30,7 +30,7 @@ openssl_csr: path: /etc/ssl/prometheus/ca/csr.pem privatekey_path: /etc/ssl/prometheus/ca/key.pem - CN: "CA for promethues zone {{ promethues_zone_name }}" + CN: "CA for prometheus zone {{ prometheus_zone_name }}" useCommonNameForSAN: no key_usage: - cRLSign diff --git a/roles/monitoring/prometheus/exporter/base/tasks/tls.yml b/roles/monitoring/prometheus/exporter/base/tasks/tls.yml index 72186acb..2f880e6a 100644 --- a/roles/monitoring/prometheus/exporter/base/tasks/tls.yml +++ b/roles/monitoring/prometheus/exporter/base/tasks/tls.yml @@ -70,7 +70,7 @@ register: prometheus_exporter_server_cert_current - name: generate exporter certificate - delegate_to: "{{ promethues_server }}" + delegate_to: "{{ prometheus_server }}" community.crypto.x509_certificate_pipe: content: "{{ prometheus_exporter_server_cert_current.content | default('') | b64decode }}" csr_content: "{{ prometheus_exporter_server_csr.content | b64decode }}" @@ -89,7 +89,7 @@ notify: restart prometheus-exporter-exporter - name: slurp CA certificate - delegate_to: "{{ promethues_server }}" + delegate_to: "{{ prometheus_server }}" slurp: src: /etc/ssl/prometheus/ca-crt.pem register: prometheus_exporter_ca_certificate diff --git a/roles/monitoring/prometheus/server/filter_plugins/prometheus.py b/roles/monitoring/prometheus/server/filter_plugins/prometheus.py new file mode 100644 index 00000000..81cfae70 --- /dev/null +++ b/roles/monitoring/prometheus/server/filter_plugins/prometheus.py @@ -0,0 +1,29 @@ +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +from functools import partial + +from ansible import errors + + +def prometheus_job_targets(hostvars, jobs, targets): + try: + result = [] + for job in jobs: + for target in targets: + enabled = job in hostvars[target]['prometheus_exporters_default'] or job in hostvars[target]['prometheus_exporters_extra'] + result.append({'job': job, 'target': target, 'enabled': enabled}) + return result + except Exception as e: + raise errors.AnsibleFilterError("prometheus_job_targets(): %s" % str(e)) + + +class FilterModule(object): + + ''' prometheus filters ''' + filter_map = { + 'prometheus_job_targets': prometheus_job_targets, + } + + def filters(self): + return self.filter_map diff --git a/roles/monitoring/prometheus/server/tasks/main.yml b/roles/monitoring/prometheus/server/tasks/main.yml index 492e8dc2..44f0800e 100644 --- a/roles/monitoring/prometheus/server/tasks/main.yml +++ b/roles/monitoring/prometheus/server/tasks/main.yml @@ -54,12 +54,19 @@ loop: "{{ prometheus_zone_targets }}" copy: content: | - - targets: [ "{{ hostvars[item].ansible_default_ipv4.address }}:9999" ] + - targets: [ "{{ hostvars[item].prometheus_scrape_endpoint }}" ] labels: instance: "{{ item }}" dest: "/etc/prometheus/targets/{{ item }}.yml" -# TODO: enable targets for configured jobs using symlinks in /etc/prometheus/jobs/*/ +- name: enable targets for jobs + loop: "{{ hostvars | prometheus_job_targets(prometheus_server_jobs, prometheus_zone_targets) }}" + loop_control: + label: "{{ item.job }} -> {{ item.target }}" + file: + src: "{{ item.enabled | ternary('/etc/prometheus/targets/' + item.target + '.yml', omit) }}" + path: "/etc/prometheus/jobs/{{ item.job }}/{{ item.target }}.yml" + state: "{{ item.enabled | ternary('link', 'absent') }}" - name: generate configuration file template: -- cgit v1.2.3 From 96a0a80a8b9d79099aba971412c698179093452d Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Fri, 18 Jun 2021 01:43:09 +0200 Subject: cosmetic fix --- roles/monitoring/prometheus/exporter/base/tasks/tls.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'roles/monitoring/prometheus/exporter/base') diff --git a/roles/monitoring/prometheus/exporter/base/tasks/tls.yml b/roles/monitoring/prometheus/exporter/base/tasks/tls.yml index 2f880e6a..e34025e4 100644 --- a/roles/monitoring/prometheus/exporter/base/tasks/tls.yml +++ b/roles/monitoring/prometheus/exporter/base/tasks/tls.yml @@ -55,7 +55,7 @@ path: /etc/ssl/prometheus/exporter/crt.pem register: prometheus_exporter_server_cert -- name: read exporter client certificate issuer key id and validity +- name: read exporter client certificate validity when: prometheus_exporter_server_cert.stat.exists openssl_certificate_info: path: /etc/ssl/prometheus/exporter/crt.pem -- cgit v1.2.3