From e29ce4fdbe2ce669c62777fffa18ae8557e54a73 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Sun, 30 May 2021 22:28:46 +0200 Subject: prometheus: initial simple server role --- inventory/group_vars/chaos-at-home/network.yml | 3 +++ 1 file changed, 3 insertions(+) (limited to 'inventory/group_vars') diff --git a/inventory/group_vars/chaos-at-home/network.yml b/inventory/group_vars/chaos-at-home/network.yml index db345b75..fa34a7a0 100644 --- a/inventory/group_vars/chaos-at-home/network.yml +++ b/inventory/group_vars/chaos-at-home/network.yml @@ -41,6 +41,7 @@ network_zones: key: "{{ vault_wifi_keys.iot }}" offsets: ch-wled-test: 1 + ch-mon: 230 ch-iot: 254 svc: @@ -63,6 +64,7 @@ network_zones: ch-nic: 53 __svc_http__: 80 __svc_imap__: 143 + ch-mon: 230 ch-router-obsd: 253 ch-router: 254 ############# @@ -83,6 +85,7 @@ network_zones: ch-sw1: 201 ch-ap0: 220 ch-ap1: 221 + ch-mon: 230 ch-gnocchi: 240 ch-router: 241 -- cgit v1.2.3 From 8ab24a10ac669ade61761d37e68207b402bc277c Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Sun, 6 Jun 2021 14:57:25 +0200 Subject: prometheus: move CA to seperate role and add prometheus zone groups --- chaos-at-home/ch-mon.yml | 3 +- .../group_vars/promzone-chaos-at-home/vars.yml | 3 ++ inventory/hosts.ini | 10 ++++ roles/monitoring/prometheus/ca/tasks/main.yml | 52 ++++++++++++++++++++ roles/monitoring/prometheus/server/tasks/tls.yml | 55 ++++------------------ 5 files changed, 76 insertions(+), 47 deletions(-) create mode 100644 inventory/group_vars/promzone-chaos-at-home/vars.yml create mode 100644 roles/monitoring/prometheus/ca/tasks/main.yml (limited to 'inventory/group_vars') diff --git a/chaos-at-home/ch-mon.yml b/chaos-at-home/ch-mon.yml index a1179204..bce4adab 100644 --- a/chaos-at-home/ch-mon.yml +++ b/chaos-at-home/ch-mon.yml @@ -9,7 +9,8 @@ - role: core/ntp - role: storage/lvm/groups - role: apt-repo/spreadspace - - role: monitoring/prometheus/server + - role: monitoring/prometheus/ca - role: monitoring/prometheus/exporter/base - role: monitoring/prometheus/exporter/node - role: monitoring/prometheus/exporter/blackbox + - role: monitoring/prometheus/server diff --git a/inventory/group_vars/promzone-chaos-at-home/vars.yml b/inventory/group_vars/promzone-chaos-at-home/vars.yml new file mode 100644 index 00000000..413a6502 --- /dev/null +++ b/inventory/group_vars/promzone-chaos-at-home/vars.yml @@ -0,0 +1,3 @@ +--- +promethues_server: ch-mon +promethues_zone_name: chaos@home diff --git a/inventory/hosts.ini b/inventory/hosts.ini index 954e9374..1c1051aa 100644 --- a/inventory/hosts.ini +++ b/inventory/hosts.ini @@ -379,6 +379,16 @@ vmhost-sk-2019vm-guests vmhost-sk-tomnext-guests +## prometheus monitoring +[promzone-chaos-at-home-server] +ch-mon +[promzone-chaos-at-home] +ch-mon +ch-testvm-prometheus +[promzone-chaos-at-home:children] +promzone-chaos-at-home-server + + ## hoster [hroot] sk-2019 diff --git a/roles/monitoring/prometheus/ca/tasks/main.yml b/roles/monitoring/prometheus/ca/tasks/main.yml new file mode 100644 index 00000000..9f166321 --- /dev/null +++ b/roles/monitoring/prometheus/ca/tasks/main.yml @@ -0,0 +1,52 @@ +--- +- name: install python-cryptoraphy + apt: + name: "{{ python_basename }}-cryptography" + state: present + +- name: create base directory + file: + path: /etc/ssl/prometheus + state: directory + +- name: create CA directory + file: + path: /etc/ssl/prometheus/ca + state: directory + owner: root + group: root + mode: 0700 + +- name: create CA private key + openssl_privatekey: + path: /etc/ssl/prometheus/ca/key.pem + type: RSA + size: 4096 + owner: root + group: root + mode: 0600 + +- name: create signing request for CA certificate + openssl_csr: + path: /etc/ssl/prometheus/ca/csr.pem + privatekey_path: /etc/ssl/prometheus/ca/key.pem + CN: "CA for promethues zone {{ promethues_zone_name }}" + useCommonNameForSAN: no + key_usage: + - cRLSign + - digitalSignature + - keyCertSign + key_usage_critical: yes + basic_constraints: + - 'CA:TRUE' + - 'pathlen:0' + basic_constraints_critical: yes + +- name: create self-signed CA certificate + openssl_certificate: + path: /etc/ssl/prometheus/ca-crt.pem + csr_path: /etc/ssl/prometheus/ca/csr.pem + privatekey_path: /etc/ssl/prometheus/ca/key.pem + provider: selfsigned + selfsigned_digest: sha256 + selfsigned_not_after: "+18250d" ## 50 years diff --git a/roles/monitoring/prometheus/server/tasks/tls.yml b/roles/monitoring/prometheus/server/tasks/tls.yml index f9ad5ca3..5c112e12 100644 --- a/roles/monitoring/prometheus/server/tasks/tls.yml +++ b/roles/monitoring/prometheus/server/tasks/tls.yml @@ -9,14 +9,6 @@ path: /etc/ssl/prometheus state: directory -- name: create CA directory - file: - path: /etc/ssl/prometheus/ca - state: directory - owner: root - group: root - mode: 0700 - - name: create server cert/key directory file: path: /etc/ssl/prometheus/server @@ -25,42 +17,7 @@ group: prometheus mode: 0750 -- name: create CA private key - openssl_privatekey: - path: /etc/ssl/prometheus/ca/key.pem - type: RSA - size: 4096 - owner: root - group: root - mode: 0600 - -- name: create signing request for CA certificate - openssl_csr: - path: /etc/ssl/prometheus/ca/csr.pem - privatekey_path: /etc/ssl/prometheus/ca/key.pem - CN: "prometheus CA" - useCommonNameForSAN: no - key_usage: - - cRLSign - - digitalSignature - - keyCertSign - key_usage_critical: yes - basic_constraints: - - 'CA:TRUE' - - 'pathlen:0' - basic_constraints_critical: yes - -- name: create self-signed CA certificate - openssl_certificate: - path: /etc/ssl/prometheus/ca-crt.pem - csr_path: /etc/ssl/prometheus/ca/csr.pem - privatekey_path: /etc/ssl/prometheus/ca/key.pem - provider: selfsigned - selfsigned_digest: sha256 - selfsigned_not_after: "+18250d" ## 50 years - - -- name: create server private key to connect to exporter +- name: create private key to connect to exporter openssl_privatekey: path: /etc/ssl/prometheus/server/exporter-key.pem type: RSA @@ -68,8 +25,9 @@ owner: prometheus group: prometheus mode: 0400 + notify: reload prometheus -- name: create signing request for server certificate to connect to exporter +- name: create signing request for client certificate to connect to exporter openssl_csr: path: /etc/ssl/prometheus/server/exporter-csr.pem privatekey_path: /etc/ssl/prometheus/server/exporter-key.pem @@ -87,7 +45,9 @@ - 'CA:FALSE' basic_constraints_critical: yes -- name: create server certificate to connect to exporter +## TODO: implement remote signing? + +- name: create client certificate to connect to exporter openssl_certificate: path: /etc/ssl/prometheus/server/exporter-crt.pem csr_path: /etc/ssl/prometheus/server/exporter-csr.pem @@ -96,3 +56,6 @@ ownca_privatekey_path: /etc/ssl/prometheus/ca/key.pem ownca_digest: sha256 ownca_not_after: "+18250d" ## 50 years + notify: reload prometheus + +## TODO: install /etc/ssl/prometheus/ca-crt.pem from server -- cgit v1.2.3 From 11baa089a6aaf62a5c35f8009aebf889a4bf85fa Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Thu, 10 Jun 2021 01:29:39 +0200 Subject: prometheus: generate target configs --- inventory/group_vars/promzone-chaos-at-home/vars.yml | 2 ++ roles/monitoring/prometheus/server/tasks/main.yml | 11 +++++++++++ 2 files changed, 13 insertions(+) (limited to 'inventory/group_vars') diff --git a/inventory/group_vars/promzone-chaos-at-home/vars.yml b/inventory/group_vars/promzone-chaos-at-home/vars.yml index 413a6502..8a0d0aa8 100644 --- a/inventory/group_vars/promzone-chaos-at-home/vars.yml +++ b/inventory/group_vars/promzone-chaos-at-home/vars.yml @@ -1,3 +1,5 @@ --- promethues_server: ch-mon promethues_zone_name: chaos@home + +prometheus_zone_targets: "{{ groups['promzone-chaos-at-home'] }}" diff --git a/roles/monitoring/prometheus/server/tasks/main.yml b/roles/monitoring/prometheus/server/tasks/main.yml index 6b030fb4..492e8dc2 100644 --- a/roles/monitoring/prometheus/server/tasks/main.yml +++ b/roles/monitoring/prometheus/server/tasks/main.yml @@ -50,6 +50,17 @@ - name: create TLS CA and certificates import_tasks: tls.yml +- name: generate targets config + loop: "{{ prometheus_zone_targets }}" + copy: + content: | + - targets: [ "{{ hostvars[item].ansible_default_ipv4.address }}:9999" ] + labels: + instance: "{{ item }}" + dest: "/etc/prometheus/targets/{{ item }}.yml" + +# TODO: enable targets for configured jobs using symlinks in /etc/prometheus/jobs/*/ + - name: generate configuration file template: src: prometheus.yml.j2 -- cgit v1.2.3 From 4e5f835b6dd5aee26a663155211ee5dd3642d07d Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Mon, 14 Jun 2021 00:49:48 +0200 Subject: make prometheus exporter list groupvars --- inventory/group_vars/promzone-chaos-at-home/vars.yml | 4 ++++ inventory/host_vars/ch-mon.yml | 3 +-- roles/monitoring/prometheus/exporter/defaults/main.yml | 3 --- roles/monitoring/prometheus/exporter/meta/main.yml | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) delete mode 100644 roles/monitoring/prometheus/exporter/defaults/main.yml (limited to 'inventory/group_vars') diff --git a/inventory/group_vars/promzone-chaos-at-home/vars.yml b/inventory/group_vars/promzone-chaos-at-home/vars.yml index 8a0d0aa8..2345292b 100644 --- a/inventory/group_vars/promzone-chaos-at-home/vars.yml +++ b/inventory/group_vars/promzone-chaos-at-home/vars.yml @@ -3,3 +3,7 @@ promethues_server: ch-mon promethues_zone_name: chaos@home prometheus_zone_targets: "{{ groups['promzone-chaos-at-home'] }}" + +prometheus_exporters_extra: [] +prometheus_exporters_default: + - node diff --git a/inventory/host_vars/ch-mon.yml b/inventory/host_vars/ch-mon.yml index 25dae3ac..025289a4 100644 --- a/inventory/host_vars/ch-mon.yml +++ b/inventory/host_vars/ch-mon.yml @@ -61,8 +61,7 @@ prometheus_server_storage: size: 30G fs: ext4 -prometheus_exporters: - - node +prometheus_exporters_extra: - blackbox prometheus_exporter_blackbox_modules_extra: diff --git a/roles/monitoring/prometheus/exporter/defaults/main.yml b/roles/monitoring/prometheus/exporter/defaults/main.yml deleted file mode 100644 index 858c1837..00000000 --- a/roles/monitoring/prometheus/exporter/defaults/main.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -prometheus_exporters: - - node diff --git a/roles/monitoring/prometheus/exporter/meta/main.yml b/roles/monitoring/prometheus/exporter/meta/main.yml index ddb30f9a..d1d3eac7 100644 --- a/roles/monitoring/prometheus/exporter/meta/main.yml +++ b/roles/monitoring/prometheus/exporter/meta/main.yml @@ -2,6 +2,6 @@ dependencies: - role: monitoring/prometheus/exporter/base - role: monitoring/prometheus/exporter/node - when: "'node' in prometheus_exporters" + when: "'node' in (prometheus_exporters_default | union(prometheus_exporters_extra))" - role: monitoring/prometheus/exporter/blackbox - when: "'blackbox' in prometheus_exporters" + when: "'blackbox' in (prometheus_exporters_default | union(prometheus_exporters_extra))" -- cgit v1.2.3 From 1e9d610bb87ce6f0cb1e5a8d44f09616f90273e2 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Fri, 18 Jun 2021 01:24:40 +0200 Subject: prometheus enable/disable targets for jobs --- .../group_vars/promzone-chaos-at-home/vars.yml | 12 ++++++--- roles/monitoring/prometheus/ca/tasks/main.yml | 2 +- .../prometheus/exporter/base/tasks/tls.yml | 4 +-- .../prometheus/server/filter_plugins/prometheus.py | 29 ++++++++++++++++++++++ roles/monitoring/prometheus/server/tasks/main.yml | 11 ++++++-- 5 files changed, 49 insertions(+), 9 deletions(-) create mode 100644 roles/monitoring/prometheus/server/filter_plugins/prometheus.py (limited to 'inventory/group_vars') diff --git a/inventory/group_vars/promzone-chaos-at-home/vars.yml b/inventory/group_vars/promzone-chaos-at-home/vars.yml index 2345292b..078576f1 100644 --- a/inventory/group_vars/promzone-chaos-at-home/vars.yml +++ b/inventory/group_vars/promzone-chaos-at-home/vars.yml @@ -1,9 +1,13 @@ --- -promethues_server: ch-mon -promethues_zone_name: chaos@home - -prometheus_zone_targets: "{{ groups['promzone-chaos-at-home'] }}" +prometheus_scrape_endpoint: "{{ network.primary.address | ipaddr('address') }}:9999" prometheus_exporters_extra: [] prometheus_exporters_default: - node + +prometheus_server: ch-mon +prometheus_server_jobs: + - node + +prometheus_zone_name: chaos@home +prometheus_zone_targets: "{{ groups['promzone-chaos-at-home'] }}" diff --git a/roles/monitoring/prometheus/ca/tasks/main.yml b/roles/monitoring/prometheus/ca/tasks/main.yml index cde4a267..064cb6e8 100644 --- a/roles/monitoring/prometheus/ca/tasks/main.yml +++ b/roles/monitoring/prometheus/ca/tasks/main.yml @@ -30,7 +30,7 @@ openssl_csr: path: /etc/ssl/prometheus/ca/csr.pem privatekey_path: /etc/ssl/prometheus/ca/key.pem - CN: "CA for promethues zone {{ promethues_zone_name }}" + CN: "CA for prometheus zone {{ prometheus_zone_name }}" useCommonNameForSAN: no key_usage: - cRLSign diff --git a/roles/monitoring/prometheus/exporter/base/tasks/tls.yml b/roles/monitoring/prometheus/exporter/base/tasks/tls.yml index 72186acb..2f880e6a 100644 --- a/roles/monitoring/prometheus/exporter/base/tasks/tls.yml +++ b/roles/monitoring/prometheus/exporter/base/tasks/tls.yml @@ -70,7 +70,7 @@ register: prometheus_exporter_server_cert_current - name: generate exporter certificate - delegate_to: "{{ promethues_server }}" + delegate_to: "{{ prometheus_server }}" community.crypto.x509_certificate_pipe: content: "{{ prometheus_exporter_server_cert_current.content | default('') | b64decode }}" csr_content: "{{ prometheus_exporter_server_csr.content | b64decode }}" @@ -89,7 +89,7 @@ notify: restart prometheus-exporter-exporter - name: slurp CA certificate - delegate_to: "{{ promethues_server }}" + delegate_to: "{{ prometheus_server }}" slurp: src: /etc/ssl/prometheus/ca-crt.pem register: prometheus_exporter_ca_certificate diff --git a/roles/monitoring/prometheus/server/filter_plugins/prometheus.py b/roles/monitoring/prometheus/server/filter_plugins/prometheus.py new file mode 100644 index 00000000..81cfae70 --- /dev/null +++ b/roles/monitoring/prometheus/server/filter_plugins/prometheus.py @@ -0,0 +1,29 @@ +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +from functools import partial + +from ansible import errors + + +def prometheus_job_targets(hostvars, jobs, targets): + try: + result = [] + for job in jobs: + for target in targets: + enabled = job in hostvars[target]['prometheus_exporters_default'] or job in hostvars[target]['prometheus_exporters_extra'] + result.append({'job': job, 'target': target, 'enabled': enabled}) + return result + except Exception as e: + raise errors.AnsibleFilterError("prometheus_job_targets(): %s" % str(e)) + + +class FilterModule(object): + + ''' prometheus filters ''' + filter_map = { + 'prometheus_job_targets': prometheus_job_targets, + } + + def filters(self): + return self.filter_map diff --git a/roles/monitoring/prometheus/server/tasks/main.yml b/roles/monitoring/prometheus/server/tasks/main.yml index 492e8dc2..44f0800e 100644 --- a/roles/monitoring/prometheus/server/tasks/main.yml +++ b/roles/monitoring/prometheus/server/tasks/main.yml @@ -54,12 +54,19 @@ loop: "{{ prometheus_zone_targets }}" copy: content: | - - targets: [ "{{ hostvars[item].ansible_default_ipv4.address }}:9999" ] + - targets: [ "{{ hostvars[item].prometheus_scrape_endpoint }}" ] labels: instance: "{{ item }}" dest: "/etc/prometheus/targets/{{ item }}.yml" -# TODO: enable targets for configured jobs using symlinks in /etc/prometheus/jobs/*/ +- name: enable targets for jobs + loop: "{{ hostvars | prometheus_job_targets(prometheus_server_jobs, prometheus_zone_targets) }}" + loop_control: + label: "{{ item.job }} -> {{ item.target }}" + file: + src: "{{ item.enabled | ternary('/etc/prometheus/targets/' + item.target + '.yml', omit) }}" + path: "/etc/prometheus/jobs/{{ item.job }}/{{ item.target }}.yml" + state: "{{ item.enabled | ternary('link', 'absent') }}" - name: generate configuration file template: -- cgit v1.2.3