From 8ab24a10ac669ade61761d37e68207b402bc277c Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Sun, 6 Jun 2021 14:57:25 +0200 Subject: prometheus: move CA to seperate role and add prometheus zone groups --- roles/monitoring/prometheus/ca/tasks/main.yml | 52 +++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 roles/monitoring/prometheus/ca/tasks/main.yml (limited to 'roles/monitoring/prometheus/ca/tasks/main.yml') diff --git a/roles/monitoring/prometheus/ca/tasks/main.yml b/roles/monitoring/prometheus/ca/tasks/main.yml new file mode 100644 index 00000000..9f166321 --- /dev/null +++ b/roles/monitoring/prometheus/ca/tasks/main.yml @@ -0,0 +1,52 @@ +--- +- name: install python-cryptoraphy + apt: + name: "{{ python_basename }}-cryptography" + state: present + +- name: create base directory + file: + path: /etc/ssl/prometheus + state: directory + +- name: create CA directory + file: + path: /etc/ssl/prometheus/ca + state: directory + owner: root + group: root + mode: 0700 + +- name: create CA private key + openssl_privatekey: + path: /etc/ssl/prometheus/ca/key.pem + type: RSA + size: 4096 + owner: root + group: root + mode: 0600 + +- name: create signing request for CA certificate + openssl_csr: + path: /etc/ssl/prometheus/ca/csr.pem + privatekey_path: /etc/ssl/prometheus/ca/key.pem + CN: "CA for promethues zone {{ promethues_zone_name }}" + useCommonNameForSAN: no + key_usage: + - cRLSign + - digitalSignature + - keyCertSign + key_usage_critical: yes + basic_constraints: + - 'CA:TRUE' + - 'pathlen:0' + basic_constraints_critical: yes + +- name: create self-signed CA certificate + openssl_certificate: + path: /etc/ssl/prometheus/ca-crt.pem + csr_path: /etc/ssl/prometheus/ca/csr.pem + privatekey_path: /etc/ssl/prometheus/ca/key.pem + provider: selfsigned + selfsigned_digest: sha256 + selfsigned_not_after: "+18250d" ## 50 years -- cgit v1.2.3 From 6082a92fa86d121d3ea4256859ee4c9d412e78c0 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Thu, 10 Jun 2021 01:15:32 +0200 Subject: promethues: remote certificate signing for exporter/base --- chaos-at-home/ch-testvm-prometheus.yml | 7 +++- inventory/host_vars/ch-testvm-prometheus.yml | 3 ++ roles/monitoring/prometheus/ca/tasks/main.yml | 2 +- .../prometheus/exporter/base/tasks/tls.yml | 49 +++++++++++++++++++--- roles/monitoring/prometheus/server/tasks/tls.yml | 34 ++++++++++----- .../prometheus/server/templates/prometheus.yml.j2 | 16 +++---- 6 files changed, 85 insertions(+), 26 deletions(-) (limited to 'roles/monitoring/prometheus/ca/tasks/main.yml') diff --git a/chaos-at-home/ch-testvm-prometheus.yml b/chaos-at-home/ch-testvm-prometheus.yml index a34d58e3..9caa2f9a 100644 --- a/chaos-at-home/ch-testvm-prometheus.yml +++ b/chaos-at-home/ch-testvm-prometheus.yml @@ -7,5 +7,8 @@ - role: core/sshd/base - role: core/zsh - role: core/ntp - - role: kubernetes/base - - role: kubernetes/standalone/base + - role: apt-repo/spreadspace + - role: monitoring/prometheus/exporter/base + - role: monitoring/prometheus/exporter/node + # - role: kubernetes/base + # - role: kubernetes/standalone/base diff --git a/inventory/host_vars/ch-testvm-prometheus.yml b/inventory/host_vars/ch-testvm-prometheus.yml index d11d565c..e539735f 100644 --- a/inventory/host_vars/ch-testvm-prometheus.yml +++ b/inventory/host_vars/ch-testvm-prometheus.yml @@ -33,6 +33,9 @@ network: - *_network_primary_ +spreadspace_apt_repo_components: + - prometheus + containerd_storage: type: lvm diff --git a/roles/monitoring/prometheus/ca/tasks/main.yml b/roles/monitoring/prometheus/ca/tasks/main.yml index 9f166321..cde4a267 100644 --- a/roles/monitoring/prometheus/ca/tasks/main.yml +++ b/roles/monitoring/prometheus/ca/tasks/main.yml @@ -34,7 +34,6 @@ useCommonNameForSAN: no key_usage: - cRLSign - - digitalSignature - keyCertSign key_usage_critical: yes basic_constraints: @@ -50,3 +49,4 @@ provider: selfsigned selfsigned_digest: sha256 selfsigned_not_after: "+18250d" ## 50 years + selfsigned_create_subject_key_identifier: always_create diff --git a/roles/monitoring/prometheus/exporter/base/tasks/tls.yml b/roles/monitoring/prometheus/exporter/base/tasks/tls.yml index b2731b09..72186acb 100644 --- a/roles/monitoring/prometheus/exporter/base/tasks/tls.yml +++ b/roles/monitoring/prometheus/exporter/base/tasks/tls.yml @@ -45,17 +45,56 @@ - 'CA:FALSE' basic_constraints_critical: yes -## TODO: implement remote singing using server +- name: slurp CSR + slurp: + src: /etc/ssl/prometheus/exporter/csr.pem + register: prometheus_exporter_server_csr -- name: create exporter certificate - openssl_certificate: +- name: check if exporter certificate exists + stat: path: /etc/ssl/prometheus/exporter/crt.pem - csr_path: /etc/ssl/prometheus/exporter/csr.pem + register: prometheus_exporter_server_cert + +- name: read exporter client certificate issuer key id and validity + when: prometheus_exporter_server_cert.stat.exists + openssl_certificate_info: + path: /etc/ssl/prometheus/exporter/crt.pem + valid_at: + ten_years: '+3650d' + register: prometheus_exporter_server_cert_info + +- name: slurp existing exporter certificate + when: prometheus_exporter_server_cert.stat.exists + slurp: + src: /etc/ssl/prometheus/exporter/crt.pem + register: prometheus_exporter_server_cert_current + +- name: generate exporter certificate + delegate_to: "{{ promethues_server }}" + community.crypto.x509_certificate_pipe: + content: "{{ prometheus_exporter_server_cert_current.content | default('') | b64decode }}" + csr_content: "{{ prometheus_exporter_server_csr.content | b64decode }}" provider: ownca ownca_path: /etc/ssl/prometheus/ca-crt.pem ownca_privatekey_path: /etc/ssl/prometheus/ca/key.pem ownca_digest: sha256 ownca_not_after: "+18250d" ## 50 years + force: "{{ prometheus_exporter_server_cert.stat.exists and (not prometheus_exporter_server_cert_info.valid_at.ten_years) }}" + register: prometheus_exporter_server_cert + +- name: store exporter certificate + copy: + content: "{{ prometheus_exporter_server_cert.certificate }}" + dest: /etc/ssl/prometheus/exporter/crt.pem notify: restart prometheus-exporter-exporter -## TODO: install /etc/ssl/prometheus/ca-crt.pem from server +- name: slurp CA certificate + delegate_to: "{{ promethues_server }}" + slurp: + src: /etc/ssl/prometheus/ca-crt.pem + register: prometheus_exporter_ca_certificate + +- name: install CA certificate + copy: + content: "{{ prometheus_exporter_ca_certificate.content | b64decode }}" + dest: /etc/ssl/prometheus/ca-crt.pem diff --git a/roles/monitoring/prometheus/server/tasks/tls.yml b/roles/monitoring/prometheus/server/tasks/tls.yml index 5c112e12..940c69b1 100644 --- a/roles/monitoring/prometheus/server/tasks/tls.yml +++ b/roles/monitoring/prometheus/server/tasks/tls.yml @@ -17,9 +17,9 @@ group: prometheus mode: 0750 -- name: create private key to connect to exporter +- name: create private key for scrape-client certificate openssl_privatekey: - path: /etc/ssl/prometheus/server/exporter-key.pem + path: /etc/ssl/prometheus/server/scrape-key.pem type: RSA size: 4096 owner: prometheus @@ -27,10 +27,10 @@ mode: 0400 notify: reload prometheus -- name: create signing request for client certificate to connect to exporter +- name: create signing request for scrape-client certificate openssl_csr: - path: /etc/ssl/prometheus/server/exporter-csr.pem - privatekey_path: /etc/ssl/prometheus/server/exporter-key.pem + path: /etc/ssl/prometheus/server/scrape-csr.pem + privatekey_path: /etc/ssl/prometheus/server/scrape-key.pem CN: "{{ inventory_hostname }}" subject_alt_name: - "DNS:{{ host_name }}.{{ host_domain }}" @@ -45,17 +45,31 @@ - 'CA:FALSE' basic_constraints_critical: yes +## TODO: install /etc/ssl/prometheus/ca-crt.pem from CA host + +- name: check if scrape-client certificate exists + stat: + path: /etc/ssl/prometheus/server/scrape-crt.pem + register: prometheus_server_scrape_client_cert + +- name: check scrape-client certificate validity + when: prometheus_server_scrape_client_cert.stat.exists + openssl_certificate_info: + path: /etc/ssl/prometheus/server/scrape-crt.pem + valid_at: + ten_years: '+3650d' + register: prometheus_server_scrape_client_cert_info + ## TODO: implement remote signing? -- name: create client certificate to connect to exporter +- name: create scrape-client certificate openssl_certificate: - path: /etc/ssl/prometheus/server/exporter-crt.pem - csr_path: /etc/ssl/prometheus/server/exporter-csr.pem + path: /etc/ssl/prometheus/server/scrape-crt.pem + csr_path: /etc/ssl/prometheus/server/scrape-csr.pem provider: ownca ownca_path: /etc/ssl/prometheus/ca-crt.pem ownca_privatekey_path: /etc/ssl/prometheus/ca/key.pem ownca_digest: sha256 ownca_not_after: "+18250d" ## 50 years + force: "{{ prometheus_server_scrape_client_cert.stat.exists and (not prometheus_server_scrape_client_cert_info.valid_at.ten_years) }}" notify: reload prometheus - -## TODO: install /etc/ssl/prometheus/ca-crt.pem from server diff --git a/roles/monitoring/prometheus/server/templates/prometheus.yml.j2 b/roles/monitoring/prometheus/server/templates/prometheus.yml.j2 index 5eb7c570..3975c74d 100644 --- a/roles/monitoring/prometheus/server/templates/prometheus.yml.j2 +++ b/roles/monitoring/prometheus/server/templates/prometheus.yml.j2 @@ -23,8 +23,8 @@ scrape_configs: scheme: https tls_config: ca_file: /etc/ssl/prometheus/ca-crt.pem - cert_file: /etc/ssl/prometheus/server/exporter-crt.pem - key_file: /etc/ssl/prometheus/server/exporter-key.pem + cert_file: /etc/ssl/prometheus/server/scrape-crt.pem + key_file: /etc/ssl/prometheus/server/scrape-key.pem file_sd_configs: - files: - "/etc/prometheus/jobs/{{ job }}/*.yml" @@ -40,8 +40,8 @@ scrape_configs: scheme: https tls_config: ca_file: /etc/ssl/prometheus/ca-crt.pem - cert_file: /etc/ssl/prometheus/server/exporter-crt.pem - key_file: /etc/ssl/prometheus/server/exporter-key.pem + cert_file: /etc/ssl/prometheus/server/scrape-crt.pem + key_file: /etc/ssl/prometheus/server/scrape-key.pem static_configs: - targets: - 62.99.185.129 @@ -63,8 +63,8 @@ scrape_configs: scheme: https tls_config: ca_file: /etc/ssl/prometheus/ca-crt.pem - cert_file: /etc/ssl/prometheus/server/exporter-crt.pem - key_file: /etc/ssl/prometheus/server/exporter-key.pem + cert_file: /etc/ssl/prometheus/server/scrape-crt.pem + key_file: /etc/ssl/prometheus/server/scrape-key.pem static_configs: - targets: - web.chaos-at-home.org @@ -85,8 +85,8 @@ scrape_configs: scheme: https tls_config: ca_file: /etc/ssl/prometheus/ca-crt.pem - cert_file: /etc/ssl/prometheus/server/exporter-crt.pem - key_file: /etc/ssl/prometheus/server/exporter-key.pem + cert_file: /etc/ssl/prometheus/server/scrape-crt.pem + key_file: /etc/ssl/prometheus/server/scrape-key.pem static_configs: - targets: - 192.168.32.230:222 -- cgit v1.2.3 From 1e9d610bb87ce6f0cb1e5a8d44f09616f90273e2 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Fri, 18 Jun 2021 01:24:40 +0200 Subject: prometheus enable/disable targets for jobs --- .../group_vars/promzone-chaos-at-home/vars.yml | 12 ++++++--- roles/monitoring/prometheus/ca/tasks/main.yml | 2 +- .../prometheus/exporter/base/tasks/tls.yml | 4 +-- .../prometheus/server/filter_plugins/prometheus.py | 29 ++++++++++++++++++++++ roles/monitoring/prometheus/server/tasks/main.yml | 11 ++++++-- 5 files changed, 49 insertions(+), 9 deletions(-) create mode 100644 roles/monitoring/prometheus/server/filter_plugins/prometheus.py (limited to 'roles/monitoring/prometheus/ca/tasks/main.yml') diff --git a/inventory/group_vars/promzone-chaos-at-home/vars.yml b/inventory/group_vars/promzone-chaos-at-home/vars.yml index 2345292b..078576f1 100644 --- a/inventory/group_vars/promzone-chaos-at-home/vars.yml +++ b/inventory/group_vars/promzone-chaos-at-home/vars.yml @@ -1,9 +1,13 @@ --- -promethues_server: ch-mon -promethues_zone_name: chaos@home - -prometheus_zone_targets: "{{ groups['promzone-chaos-at-home'] }}" +prometheus_scrape_endpoint: "{{ network.primary.address | ipaddr('address') }}:9999" prometheus_exporters_extra: [] prometheus_exporters_default: - node + +prometheus_server: ch-mon +prometheus_server_jobs: + - node + +prometheus_zone_name: chaos@home +prometheus_zone_targets: "{{ groups['promzone-chaos-at-home'] }}" diff --git a/roles/monitoring/prometheus/ca/tasks/main.yml b/roles/monitoring/prometheus/ca/tasks/main.yml index cde4a267..064cb6e8 100644 --- a/roles/monitoring/prometheus/ca/tasks/main.yml +++ b/roles/monitoring/prometheus/ca/tasks/main.yml @@ -30,7 +30,7 @@ openssl_csr: path: /etc/ssl/prometheus/ca/csr.pem privatekey_path: /etc/ssl/prometheus/ca/key.pem - CN: "CA for promethues zone {{ promethues_zone_name }}" + CN: "CA for prometheus zone {{ prometheus_zone_name }}" useCommonNameForSAN: no key_usage: - cRLSign diff --git a/roles/monitoring/prometheus/exporter/base/tasks/tls.yml b/roles/monitoring/prometheus/exporter/base/tasks/tls.yml index 72186acb..2f880e6a 100644 --- a/roles/monitoring/prometheus/exporter/base/tasks/tls.yml +++ b/roles/monitoring/prometheus/exporter/base/tasks/tls.yml @@ -70,7 +70,7 @@ register: prometheus_exporter_server_cert_current - name: generate exporter certificate - delegate_to: "{{ promethues_server }}" + delegate_to: "{{ prometheus_server }}" community.crypto.x509_certificate_pipe: content: "{{ prometheus_exporter_server_cert_current.content | default('') | b64decode }}" csr_content: "{{ prometheus_exporter_server_csr.content | b64decode }}" @@ -89,7 +89,7 @@ notify: restart prometheus-exporter-exporter - name: slurp CA certificate - delegate_to: "{{ promethues_server }}" + delegate_to: "{{ prometheus_server }}" slurp: src: /etc/ssl/prometheus/ca-crt.pem register: prometheus_exporter_ca_certificate diff --git a/roles/monitoring/prometheus/server/filter_plugins/prometheus.py b/roles/monitoring/prometheus/server/filter_plugins/prometheus.py new file mode 100644 index 00000000..81cfae70 --- /dev/null +++ b/roles/monitoring/prometheus/server/filter_plugins/prometheus.py @@ -0,0 +1,29 @@ +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +from functools import partial + +from ansible import errors + + +def prometheus_job_targets(hostvars, jobs, targets): + try: + result = [] + for job in jobs: + for target in targets: + enabled = job in hostvars[target]['prometheus_exporters_default'] or job in hostvars[target]['prometheus_exporters_extra'] + result.append({'job': job, 'target': target, 'enabled': enabled}) + return result + except Exception as e: + raise errors.AnsibleFilterError("prometheus_job_targets(): %s" % str(e)) + + +class FilterModule(object): + + ''' prometheus filters ''' + filter_map = { + 'prometheus_job_targets': prometheus_job_targets, + } + + def filters(self): + return self.filter_map diff --git a/roles/monitoring/prometheus/server/tasks/main.yml b/roles/monitoring/prometheus/server/tasks/main.yml index 492e8dc2..44f0800e 100644 --- a/roles/monitoring/prometheus/server/tasks/main.yml +++ b/roles/monitoring/prometheus/server/tasks/main.yml @@ -54,12 +54,19 @@ loop: "{{ prometheus_zone_targets }}" copy: content: | - - targets: [ "{{ hostvars[item].ansible_default_ipv4.address }}:9999" ] + - targets: [ "{{ hostvars[item].prometheus_scrape_endpoint }}" ] labels: instance: "{{ item }}" dest: "/etc/prometheus/targets/{{ item }}.yml" -# TODO: enable targets for configured jobs using symlinks in /etc/prometheus/jobs/*/ +- name: enable targets for jobs + loop: "{{ hostvars | prometheus_job_targets(prometheus_server_jobs, prometheus_zone_targets) }}" + loop_control: + label: "{{ item.job }} -> {{ item.target }}" + file: + src: "{{ item.enabled | ternary('/etc/prometheus/targets/' + item.target + '.yml', omit) }}" + path: "/etc/prometheus/jobs/{{ item.job }}/{{ item.target }}.yml" + state: "{{ item.enabled | ternary('link', 'absent') }}" - name: generate configuration file template: -- cgit v1.2.3