From a8e8cb2ed3d5e68d89edd8785ed59f0ee45f81bf Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Tue, 21 Sep 2021 19:34:25 +0200 Subject: prometheus: simplify job config --- inventory/group_vars/chaos-at-home-ups/vars.yml | 2 +- inventory/group_vars/ele-ups/vars.yml | 2 +- inventory/group_vars/promzone-chaos-at-home/vars.yml | 5 ++--- inventory/group_vars/promzone-elevate-festival/vars.yml | 3 ++- 4 files changed, 6 insertions(+), 6 deletions(-) (limited to 'inventory') diff --git a/inventory/group_vars/chaos-at-home-ups/vars.yml b/inventory/group_vars/chaos-at-home-ups/vars.yml index 5ff68452..4f003a7a 100644 --- a/inventory/group_vars/chaos-at-home-ups/vars.yml +++ b/inventory/group_vars/chaos-at-home-ups/vars.yml @@ -11,7 +11,7 @@ prometheus_scrape_endpoint: "{{ network_mgmt_zone.prefix | ipaddr(network_mgmt_z prometheus_exporters_default: - openwrt -prometheus_special_job_nut: +prometheus_special_job_nut_ups: - exporter_hostname: ch-mon instance: "ups-{{ ups_name }}" ups: "{{ ups_name }}" diff --git a/inventory/group_vars/ele-ups/vars.yml b/inventory/group_vars/ele-ups/vars.yml index cbee3ee8..a57382ff 100644 --- a/inventory/group_vars/ele-ups/vars.yml +++ b/inventory/group_vars/ele-ups/vars.yml @@ -14,7 +14,7 @@ prometheus_scrape_endpoint: "{{ network_mgmt_zone.prefix | ipaddr(network_mgmt_z prometheus_exporters_default: - openwrt -prometheus_special_job_nut: +prometheus_special_job_nut_ups: exporter_hostname: ele-mon instance: "ups-{{ ups_name }}" ups: "{{ ups_name }}" diff --git a/inventory/group_vars/promzone-chaos-at-home/vars.yml b/inventory/group_vars/promzone-chaos-at-home/vars.yml index fcb04716..d1958d47 100644 --- a/inventory/group_vars/promzone-chaos-at-home/vars.yml +++ b/inventory/group_vars/promzone-chaos-at-home/vars.yml @@ -6,11 +6,10 @@ prometheus_exporters_default: - node prometheus_server: ch-mon -prometheus_server_jobs_generic: +prometheus_server_jobs: - node - openwrt -prometheus_server_jobs_special: - - nut + - nut-ups - blackbox-ping - blackbox-https - blackbox-ssh diff --git a/inventory/group_vars/promzone-elevate-festival/vars.yml b/inventory/group_vars/promzone-elevate-festival/vars.yml index e94943d7..a65a0cb7 100644 --- a/inventory/group_vars/promzone-elevate-festival/vars.yml +++ b/inventory/group_vars/promzone-elevate-festival/vars.yml @@ -6,9 +6,10 @@ prometheus_exporters_default: - node prometheus_server: ele-mon -prometheus_server_jobs_generic: +prometheus_server_jobs: - node - openwrt + - nut-ups prometheus_zone_name: Elevate Festival prometheus_zone_targets: "{{ groups['promzone-elevate-festival'] }}" -- cgit v1.2.3 From ad08b01391c404d4e0356467fc627d711ece8916 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Wed, 22 Sep 2021 01:25:59 +0200 Subject: prometheus: no more special jobs --- inventory/group_vars/chaos-at-home-ups/vars.yml | 2 +- inventory/group_vars/ele-ups/vars.yml | 2 +- inventory/host_vars/ch-mon.yml | 6 +++--- roles/monitoring/prometheus/server/filter_plugins/prometheus.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) (limited to 'inventory') diff --git a/inventory/group_vars/chaos-at-home-ups/vars.yml b/inventory/group_vars/chaos-at-home-ups/vars.yml index 4f003a7a..99868165 100644 --- a/inventory/group_vars/chaos-at-home-ups/vars.yml +++ b/inventory/group_vars/chaos-at-home-ups/vars.yml @@ -11,7 +11,7 @@ prometheus_scrape_endpoint: "{{ network_mgmt_zone.prefix | ipaddr(network_mgmt_z prometheus_exporters_default: - openwrt -prometheus_special_job_nut_ups: +prometheus_job_nut_ups: - exporter_hostname: ch-mon instance: "ups-{{ ups_name }}" ups: "{{ ups_name }}" diff --git a/inventory/group_vars/ele-ups/vars.yml b/inventory/group_vars/ele-ups/vars.yml index a57382ff..0d22f770 100644 --- a/inventory/group_vars/ele-ups/vars.yml +++ b/inventory/group_vars/ele-ups/vars.yml @@ -14,7 +14,7 @@ prometheus_scrape_endpoint: "{{ network_mgmt_zone.prefix | ipaddr(network_mgmt_z prometheus_exporters_default: - openwrt -prometheus_special_job_nut_ups: +prometheus_job_nut_ups: exporter_hostname: ele-mon instance: "ups-{{ ups_name }}" ups: "{{ ups_name }}" diff --git a/inventory/host_vars/ch-mon.yml b/inventory/host_vars/ch-mon.yml index 7d8e334b..a211d4bb 100644 --- a/inventory/host_vars/ch-mon.yml +++ b/inventory/host_vars/ch-mon.yml @@ -76,7 +76,7 @@ prometheus_exporter_blackbox_modules_extra: icmp: prober: icmp -prometheus_special_job_blackbox_ping: +prometheus_job_blackbox_ping: - exporter_hostname: ch-mon instance: "ping-magentagw" address: 62.99.185.129 @@ -84,12 +84,12 @@ prometheus_special_job_blackbox_ping: instance: "ping-quad9" address: 9.9.9.9 -prometheus_special_job_blackbox_https: +prometheus_job_blackbox_https: - exporter_hostname: ch-mon instance: "https-web.chaos-at-home.org" address: web.chaos-at-home.org -prometheus_special_job_blackbox_ssh: +prometheus_job_blackbox_ssh: - exporter_hostname: ch-mon instance: "ssh-{{ inventory_hostname }}" address: "{{ network_zones.svc.prefix | ipaddr(network_zones.svc.offsets[inventory_hostname]) | ipaddr('address') }}:{{ ansible_port | default(22) }}" diff --git a/roles/monitoring/prometheus/server/filter_plugins/prometheus.py b/roles/monitoring/prometheus/server/filter_plugins/prometheus.py index ab865f93..056d216f 100644 --- a/roles/monitoring/prometheus/server/filter_plugins/prometheus.py +++ b/roles/monitoring/prometheus/server/filter_plugins/prometheus.py @@ -11,7 +11,7 @@ def prometheus_job_targets(hostvars, jobs, targets): result = [] for job in jobs: for target in targets: - special_config_varname = 'prometheus_special_job_' + job.replace('-', '_') + special_config_varname = 'prometheus_job_' + job.replace('-', '_') if special_config_varname in hostvars[target]: for config in hostvars[target][special_config_varname]: result.append({'job': job, 'instance': config['instance'], 'config': config, 'enabled': True}) -- cgit v1.2.3 From ef4432c51bacb5b92c03a42cb1ea7f9d837ec8b6 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Thu, 23 Sep 2021 16:50:24 +0200 Subject: use / as spereator for jobs formerly known as special --- inventory/group_vars/chaos-at-home-ups/vars.yml | 2 +- inventory/group_vars/ele-ups/vars.yml | 2 +- inventory/group_vars/promzone-chaos-at-home/vars.yml | 8 ++++---- inventory/group_vars/promzone-elevate-festival/vars.yml | 2 +- inventory/host_vars/ch-mon.yml | 6 +++--- .../monitoring/prometheus/server/defaults/main/main.yml | 8 ++++---- .../server/defaults/main/rules_blackbox-https.yml | 3 --- .../server/defaults/main/rules_blackbox-ping.yml | 3 --- .../server/defaults/main/rules_blackbox-ssh.yml | 3 --- .../server/defaults/main/rules_blackbox__https.yml | 3 +++ .../server/defaults/main/rules_blackbox__ping.yml | 3 +++ .../server/defaults/main/rules_blackbox__ssh.yml | 3 +++ .../prometheus/server/defaults/main/rules_nut-ups.yml | 3 --- .../prometheus/server/defaults/main/rules_nut__ups.yml | 3 +++ .../prometheus/server/filter_plugins/prometheus.py | 2 +- roles/monitoring/prometheus/server/tasks/main.yml | 8 +++++++- .../server/templates/job-snippets/blackbox-https.j2 | 14 -------------- .../server/templates/job-snippets/blackbox-ping.j2 | 14 -------------- .../server/templates/job-snippets/blackbox-ssh.j2 | 14 -------------- .../server/templates/job-snippets/blackbox/https.j2 | 14 ++++++++++++++ .../server/templates/job-snippets/blackbox/ping.j2 | 14 ++++++++++++++ .../server/templates/job-snippets/blackbox/ssh.j2 | 14 ++++++++++++++ .../prometheus/server/templates/job-snippets/nut-ups.j2 | 13 ------------- .../prometheus/server/templates/job-snippets/nut/ups.j2 | 13 +++++++++++++ .../prometheus/server/templates/prometheus.yml.j2 | 5 ++++- .../server/templates/targets/blackbox-https.yml.j2 | 4 ---- .../server/templates/targets/blackbox-ping.yml.j2 | 4 ---- .../server/templates/targets/blackbox-ssh.yml.j2 | 4 ---- .../server/templates/targets/blackbox/https.yml.j2 | 4 ++++ .../server/templates/targets/blackbox/ping.yml.j2 | 4 ++++ .../server/templates/targets/blackbox/ssh.yml.j2 | 4 ++++ .../prometheus/server/templates/targets/nut-ups.yml.j2 | 17 ----------------- .../prometheus/server/templates/targets/nut/ups.yml.j2 | 17 +++++++++++++++++ 33 files changed, 122 insertions(+), 113 deletions(-) delete mode 100644 roles/monitoring/prometheus/server/defaults/main/rules_blackbox-https.yml delete mode 100644 roles/monitoring/prometheus/server/defaults/main/rules_blackbox-ping.yml delete mode 100644 roles/monitoring/prometheus/server/defaults/main/rules_blackbox-ssh.yml create mode 100644 roles/monitoring/prometheus/server/defaults/main/rules_blackbox__https.yml create mode 100644 roles/monitoring/prometheus/server/defaults/main/rules_blackbox__ping.yml create mode 100644 roles/monitoring/prometheus/server/defaults/main/rules_blackbox__ssh.yml delete mode 100644 roles/monitoring/prometheus/server/defaults/main/rules_nut-ups.yml create mode 100644 roles/monitoring/prometheus/server/defaults/main/rules_nut__ups.yml delete mode 100644 roles/monitoring/prometheus/server/templates/job-snippets/blackbox-https.j2 delete mode 100644 roles/monitoring/prometheus/server/templates/job-snippets/blackbox-ping.j2 delete mode 100644 roles/monitoring/prometheus/server/templates/job-snippets/blackbox-ssh.j2 create mode 100644 roles/monitoring/prometheus/server/templates/job-snippets/blackbox/https.j2 create mode 100644 roles/monitoring/prometheus/server/templates/job-snippets/blackbox/ping.j2 create mode 100644 roles/monitoring/prometheus/server/templates/job-snippets/blackbox/ssh.j2 delete mode 100644 roles/monitoring/prometheus/server/templates/job-snippets/nut-ups.j2 create mode 100644 roles/monitoring/prometheus/server/templates/job-snippets/nut/ups.j2 delete mode 100644 roles/monitoring/prometheus/server/templates/targets/blackbox-https.yml.j2 delete mode 100644 roles/monitoring/prometheus/server/templates/targets/blackbox-ping.yml.j2 delete mode 100644 roles/monitoring/prometheus/server/templates/targets/blackbox-ssh.yml.j2 create mode 100644 roles/monitoring/prometheus/server/templates/targets/blackbox/https.yml.j2 create mode 100644 roles/monitoring/prometheus/server/templates/targets/blackbox/ping.yml.j2 create mode 100644 roles/monitoring/prometheus/server/templates/targets/blackbox/ssh.yml.j2 delete mode 100644 roles/monitoring/prometheus/server/templates/targets/nut-ups.yml.j2 create mode 100644 roles/monitoring/prometheus/server/templates/targets/nut/ups.yml.j2 (limited to 'inventory') diff --git a/inventory/group_vars/chaos-at-home-ups/vars.yml b/inventory/group_vars/chaos-at-home-ups/vars.yml index 99868165..f8c1bdf1 100644 --- a/inventory/group_vars/chaos-at-home-ups/vars.yml +++ b/inventory/group_vars/chaos-at-home-ups/vars.yml @@ -11,7 +11,7 @@ prometheus_scrape_endpoint: "{{ network_mgmt_zone.prefix | ipaddr(network_mgmt_z prometheus_exporters_default: - openwrt -prometheus_job_nut_ups: +prometheus_job_nut__ups: - exporter_hostname: ch-mon instance: "ups-{{ ups_name }}" ups: "{{ ups_name }}" diff --git a/inventory/group_vars/ele-ups/vars.yml b/inventory/group_vars/ele-ups/vars.yml index 0d22f770..1c4613a3 100644 --- a/inventory/group_vars/ele-ups/vars.yml +++ b/inventory/group_vars/ele-ups/vars.yml @@ -14,7 +14,7 @@ prometheus_scrape_endpoint: "{{ network_mgmt_zone.prefix | ipaddr(network_mgmt_z prometheus_exporters_default: - openwrt -prometheus_job_nut_ups: +prometheus_job_nut__ups: exporter_hostname: ele-mon instance: "ups-{{ ups_name }}" ups: "{{ ups_name }}" diff --git a/inventory/group_vars/promzone-chaos-at-home/vars.yml b/inventory/group_vars/promzone-chaos-at-home/vars.yml index d1958d47..84ed1263 100644 --- a/inventory/group_vars/promzone-chaos-at-home/vars.yml +++ b/inventory/group_vars/promzone-chaos-at-home/vars.yml @@ -9,10 +9,10 @@ prometheus_server: ch-mon prometheus_server_jobs: - node - openwrt - - nut-ups - - blackbox-ping - - blackbox-https - - blackbox-ssh + - nut/ups + - blackbox/ping + - blackbox/https + - blackbox/ssh prometheus_zone_name: chaos@home prometheus_zone_targets: "{{ groups['promzone-chaos-at-home'] }}" diff --git a/inventory/group_vars/promzone-elevate-festival/vars.yml b/inventory/group_vars/promzone-elevate-festival/vars.yml index a65a0cb7..43115dc4 100644 --- a/inventory/group_vars/promzone-elevate-festival/vars.yml +++ b/inventory/group_vars/promzone-elevate-festival/vars.yml @@ -9,7 +9,7 @@ prometheus_server: ele-mon prometheus_server_jobs: - node - openwrt - - nut-ups + - nut/ups prometheus_zone_name: Elevate Festival prometheus_zone_targets: "{{ groups['promzone-elevate-festival'] }}" diff --git a/inventory/host_vars/ch-mon.yml b/inventory/host_vars/ch-mon.yml index a211d4bb..242c4835 100644 --- a/inventory/host_vars/ch-mon.yml +++ b/inventory/host_vars/ch-mon.yml @@ -76,7 +76,7 @@ prometheus_exporter_blackbox_modules_extra: icmp: prober: icmp -prometheus_job_blackbox_ping: +prometheus_job_blackbox__ping: - exporter_hostname: ch-mon instance: "ping-magentagw" address: 62.99.185.129 @@ -84,12 +84,12 @@ prometheus_job_blackbox_ping: instance: "ping-quad9" address: 9.9.9.9 -prometheus_job_blackbox_https: +prometheus_job_blackbox__https: - exporter_hostname: ch-mon instance: "https-web.chaos-at-home.org" address: web.chaos-at-home.org -prometheus_job_blackbox_ssh: +prometheus_job_blackbox__ssh: - exporter_hostname: ch-mon instance: "ssh-{{ inventory_hostname }}" address: "{{ network_zones.svc.prefix | ipaddr(network_zones.svc.offsets[inventory_hostname]) | ipaddr('address') }}:{{ ansible_port | default(22) }}" diff --git a/roles/monitoring/prometheus/server/defaults/main/main.yml b/roles/monitoring/prometheus/server/defaults/main/main.yml index 5be3ecd3..bae0cdba 100644 --- a/roles/monitoring/prometheus/server/defaults/main/main.yml +++ b/roles/monitoring/prometheus/server/defaults/main/main.yml @@ -14,10 +14,10 @@ prometheus_server_rules: prometheus: "{{ prometheus_server_rules_prometheus + ((prometheus_server_alertmanager is defined) | ternary(prometheus_server_rules_prometheus_alertmanager, [])) + prometheus_server_rules_prometheus_extra }}" node: "{{ prometheus_server_rules_node + prometheus_server_rules_node_extra }}" openwrt: "{{ prometheus_server_rules_openwrt + prometheus_server_rules_node_extra }}" - "nut-ups": "{{ prometheus_server_rules_nut_ups + prometheus_server_rules_nut_ups_extra }}" - "blackbox-ping": "{{ prometheus_server_rules_blackbox_ping + prometheus_server_rules_blackbox_ping_extra }}" - "blackbox-https": "{{ prometheus_server_rules_blackbox_https + prometheus_server_rules_blackbox_https_extra }}" - "blackbox-ssh": "{{ prometheus_server_rules_blackbox_ssh + prometheus_server_rules_blackbox_ssh_extra }}" + nut/ups: "{{ prometheus_server_rules_nut__ups + prometheus_server_rules_nut__ups_extra }}" + blackbox/ping: "{{ prometheus_server_rules_blackbox__ping + prometheus_server_rules_blackbox__ping_extra }}" + blackbox/https: "{{ prometheus_server_rules_blackbox__https + prometheus_server_rules_blackbox__https_extra }}" + blackbox/ssh: "{{ prometheus_server_rules_blackbox__ssh + prometheus_server_rules_blackbox__ssh_extra }}" # prometheus_server_alertmanager: # url: "127.0.0.1:9093" diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_blackbox-https.yml b/roles/monitoring/prometheus/server/defaults/main/rules_blackbox-https.yml deleted file mode 100644 index bb806075..00000000 --- a/roles/monitoring/prometheus/server/defaults/main/rules_blackbox-https.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -prometheus_server_rules_blackbox_https_extra: [] -prometheus_server_rules_blackbox_https: [] diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_blackbox-ping.yml b/roles/monitoring/prometheus/server/defaults/main/rules_blackbox-ping.yml deleted file mode 100644 index 56c122f5..00000000 --- a/roles/monitoring/prometheus/server/defaults/main/rules_blackbox-ping.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -prometheus_server_rules_blackbox_ping_extra: [] -prometheus_server_rules_blackbox_ping: [] diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_blackbox-ssh.yml b/roles/monitoring/prometheus/server/defaults/main/rules_blackbox-ssh.yml deleted file mode 100644 index 727d2292..00000000 --- a/roles/monitoring/prometheus/server/defaults/main/rules_blackbox-ssh.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -prometheus_server_rules_blackbox_ssh_extra: [] -prometheus_server_rules_blackbox_ssh: [] diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_blackbox__https.yml b/roles/monitoring/prometheus/server/defaults/main/rules_blackbox__https.yml new file mode 100644 index 00000000..cfdc10bd --- /dev/null +++ b/roles/monitoring/prometheus/server/defaults/main/rules_blackbox__https.yml @@ -0,0 +1,3 @@ +--- +prometheus_server_rules_blackbox__https_extra: [] +prometheus_server_rules_blackbox__https: [] diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_blackbox__ping.yml b/roles/monitoring/prometheus/server/defaults/main/rules_blackbox__ping.yml new file mode 100644 index 00000000..06ce8607 --- /dev/null +++ b/roles/monitoring/prometheus/server/defaults/main/rules_blackbox__ping.yml @@ -0,0 +1,3 @@ +--- +prometheus_server_rules_blackbox__ping_extra: [] +prometheus_server_rules_blackbox__ping: [] diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_blackbox__ssh.yml b/roles/monitoring/prometheus/server/defaults/main/rules_blackbox__ssh.yml new file mode 100644 index 00000000..8e717c41 --- /dev/null +++ b/roles/monitoring/prometheus/server/defaults/main/rules_blackbox__ssh.yml @@ -0,0 +1,3 @@ +--- +prometheus_server_rules_blackbox__ssh_extra: [] +prometheus_server_rules_blackbox__ssh: [] diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_nut-ups.yml b/roles/monitoring/prometheus/server/defaults/main/rules_nut-ups.yml deleted file mode 100644 index 842007b4..00000000 --- a/roles/monitoring/prometheus/server/defaults/main/rules_nut-ups.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -prometheus_server_rules_nut_ups_extra: [] -prometheus_server_rules_nut_ups: [] diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_nut__ups.yml b/roles/monitoring/prometheus/server/defaults/main/rules_nut__ups.yml new file mode 100644 index 00000000..bccb0ca8 --- /dev/null +++ b/roles/monitoring/prometheus/server/defaults/main/rules_nut__ups.yml @@ -0,0 +1,3 @@ +--- +prometheus_server_rules_nut__ups_extra: [] +prometheus_server_rules_nut__ups: [] diff --git a/roles/monitoring/prometheus/server/filter_plugins/prometheus.py b/roles/monitoring/prometheus/server/filter_plugins/prometheus.py index 056d216f..1443e837 100644 --- a/roles/monitoring/prometheus/server/filter_plugins/prometheus.py +++ b/roles/monitoring/prometheus/server/filter_plugins/prometheus.py @@ -11,7 +11,7 @@ def prometheus_job_targets(hostvars, jobs, targets): result = [] for job in jobs: for target in targets: - special_config_varname = 'prometheus_job_' + job.replace('-', '_') + special_config_varname = 'prometheus_job_' + job.replace('-', '_').replace('/', '__') if special_config_varname in hostvars[target]: for config in hostvars[target][special_config_varname]: result.append({'job': job, 'instance': config['instance'], 'config': config, 'enabled': True}) diff --git a/roles/monitoring/prometheus/server/tasks/main.yml b/roles/monitoring/prometheus/server/tasks/main.yml index 4bcaa2d5..16167c9c 100644 --- a/roles/monitoring/prometheus/server/tasks/main.yml +++ b/roles/monitoring/prometheus/server/tasks/main.yml @@ -44,7 +44,7 @@ path: "/etc/prometheus/{{ item }}" state: directory -- name: create sub-directroy for all jobs in targets directory +- name: create sub-directories for all jobs in targets directory loop: "{{ prometheus_server_jobs }}" file: path: "/etc/prometheus/targets/{{ item }}" @@ -76,6 +76,12 @@ state: absent notify: reload prometheus +- name: create sub-directories for all jobs in rules directory + loop: "{{ prometheus_server_jobs | select('match', '.*/.*') | map('dirname') | unique }}" + file: + path: "/etc/prometheus/rules/{{ item }}" + state: directory + - name: generate rules files for all jobs loop: "{{ prometheus_server_jobs | union(['prometheus']) }}" template: diff --git a/roles/monitoring/prometheus/server/templates/job-snippets/blackbox-https.j2 b/roles/monitoring/prometheus/server/templates/job-snippets/blackbox-https.j2 deleted file mode 100644 index 98a64121..00000000 --- a/roles/monitoring/prometheus/server/templates/job-snippets/blackbox-https.j2 +++ /dev/null @@ -1,14 +0,0 @@ - - job_name: '{{ job }}' - metrics_path: /proxy - params: - module: - - blackbox - - http_tls_2xx - scheme: https - tls_config: - ca_file: /etc/ssl/prometheus/ca-crt.pem - cert_file: /etc/ssl/prometheus/server/scrape-crt.pem - key_file: /etc/ssl/prometheus/server/scrape-key.pem - file_sd_configs: - - files: - - "/etc/prometheus/targets/{{ job }}/*.yml" diff --git a/roles/monitoring/prometheus/server/templates/job-snippets/blackbox-ping.j2 b/roles/monitoring/prometheus/server/templates/job-snippets/blackbox-ping.j2 deleted file mode 100644 index 736ffec1..00000000 --- a/roles/monitoring/prometheus/server/templates/job-snippets/blackbox-ping.j2 +++ /dev/null @@ -1,14 +0,0 @@ - - job_name: '{{ job }}' - metrics_path: /proxy - params: - module: - - blackbox - - icmp - scheme: https - tls_config: - ca_file: /etc/ssl/prometheus/ca-crt.pem - cert_file: /etc/ssl/prometheus/server/scrape-crt.pem - key_file: /etc/ssl/prometheus/server/scrape-key.pem - file_sd_configs: - - files: - - "/etc/prometheus/targets/{{ job }}/*.yml" diff --git a/roles/monitoring/prometheus/server/templates/job-snippets/blackbox-ssh.j2 b/roles/monitoring/prometheus/server/templates/job-snippets/blackbox-ssh.j2 deleted file mode 100644 index 166f37ad..00000000 --- a/roles/monitoring/prometheus/server/templates/job-snippets/blackbox-ssh.j2 +++ /dev/null @@ -1,14 +0,0 @@ - - job_name: '{{ job }}' - metrics_path: /proxy - params: - module: - - blackbox - - ssh_banner - scheme: https - tls_config: - ca_file: /etc/ssl/prometheus/ca-crt.pem - cert_file: /etc/ssl/prometheus/server/scrape-crt.pem - key_file: /etc/ssl/prometheus/server/scrape-key.pem - file_sd_configs: - - files: - - "/etc/prometheus/targets/{{ job }}/*.yml" diff --git a/roles/monitoring/prometheus/server/templates/job-snippets/blackbox/https.j2 b/roles/monitoring/prometheus/server/templates/job-snippets/blackbox/https.j2 new file mode 100644 index 00000000..98a64121 --- /dev/null +++ b/roles/monitoring/prometheus/server/templates/job-snippets/blackbox/https.j2 @@ -0,0 +1,14 @@ + - job_name: '{{ job }}' + metrics_path: /proxy + params: + module: + - blackbox + - http_tls_2xx + scheme: https + tls_config: + ca_file: /etc/ssl/prometheus/ca-crt.pem + cert_file: /etc/ssl/prometheus/server/scrape-crt.pem + key_file: /etc/ssl/prometheus/server/scrape-key.pem + file_sd_configs: + - files: + - "/etc/prometheus/targets/{{ job }}/*.yml" diff --git a/roles/monitoring/prometheus/server/templates/job-snippets/blackbox/ping.j2 b/roles/monitoring/prometheus/server/templates/job-snippets/blackbox/ping.j2 new file mode 100644 index 00000000..736ffec1 --- /dev/null +++ b/roles/monitoring/prometheus/server/templates/job-snippets/blackbox/ping.j2 @@ -0,0 +1,14 @@ + - job_name: '{{ job }}' + metrics_path: /proxy + params: + module: + - blackbox + - icmp + scheme: https + tls_config: + ca_file: /etc/ssl/prometheus/ca-crt.pem + cert_file: /etc/ssl/prometheus/server/scrape-crt.pem + key_file: /etc/ssl/prometheus/server/scrape-key.pem + file_sd_configs: + - files: + - "/etc/prometheus/targets/{{ job }}/*.yml" diff --git a/roles/monitoring/prometheus/server/templates/job-snippets/blackbox/ssh.j2 b/roles/monitoring/prometheus/server/templates/job-snippets/blackbox/ssh.j2 new file mode 100644 index 00000000..166f37ad --- /dev/null +++ b/roles/monitoring/prometheus/server/templates/job-snippets/blackbox/ssh.j2 @@ -0,0 +1,14 @@ + - job_name: '{{ job }}' + metrics_path: /proxy + params: + module: + - blackbox + - ssh_banner + scheme: https + tls_config: + ca_file: /etc/ssl/prometheus/ca-crt.pem + cert_file: /etc/ssl/prometheus/server/scrape-crt.pem + key_file: /etc/ssl/prometheus/server/scrape-key.pem + file_sd_configs: + - files: + - "/etc/prometheus/targets/{{ job }}/*.yml" diff --git a/roles/monitoring/prometheus/server/templates/job-snippets/nut-ups.j2 b/roles/monitoring/prometheus/server/templates/job-snippets/nut-ups.j2 deleted file mode 100644 index 3a2c5c62..00000000 --- a/roles/monitoring/prometheus/server/templates/job-snippets/nut-ups.j2 +++ /dev/null @@ -1,13 +0,0 @@ - - job_name: '{{ job }}' - metrics_path: /proxy - params: - module: - - nut - scheme: https - tls_config: - ca_file: /etc/ssl/prometheus/ca-crt.pem - cert_file: /etc/ssl/prometheus/server/scrape-crt.pem - key_file: /etc/ssl/prometheus/server/scrape-key.pem - file_sd_configs: - - files: - - "/etc/prometheus/targets/{{ job }}/*.yml" diff --git a/roles/monitoring/prometheus/server/templates/job-snippets/nut/ups.j2 b/roles/monitoring/prometheus/server/templates/job-snippets/nut/ups.j2 new file mode 100644 index 00000000..3a2c5c62 --- /dev/null +++ b/roles/monitoring/prometheus/server/templates/job-snippets/nut/ups.j2 @@ -0,0 +1,13 @@ + - job_name: '{{ job }}' + metrics_path: /proxy + params: + module: + - nut + scheme: https + tls_config: + ca_file: /etc/ssl/prometheus/ca-crt.pem + cert_file: /etc/ssl/prometheus/server/scrape-crt.pem + key_file: /etc/ssl/prometheus/server/scrape-key.pem + file_sd_configs: + - files: + - "/etc/prometheus/targets/{{ job }}/*.yml" diff --git a/roles/monitoring/prometheus/server/templates/prometheus.yml.j2 b/roles/monitoring/prometheus/server/templates/prometheus.yml.j2 index 4a079896..8156341d 100644 --- a/roles/monitoring/prometheus/server/templates/prometheus.yml.j2 +++ b/roles/monitoring/prometheus/server/templates/prometheus.yml.j2 @@ -6,6 +6,9 @@ global: rule_files: - /etc/prometheus/rules/*.yml +{% for subdir in (prometheus_server_jobs | select('match', '.*/.*') | map('dirname') | unique) %} + - /etc/prometheus/rules/{{ subdir }}/*.yml +{% endfor %} {% if prometheus_server_alertmanager is defined %} alerting: @@ -37,7 +40,7 @@ scrape_configs: {% endif %} {% for job in (prometheus_server_jobs) %} -{% include 'job-snippets/' + (lookup('first_found', {'paths': ['templates/job-snippets'], 'files': [job + '.j2', 'generic.j2']}) | basename) %}{{ '' }} +{% include lookup('first_found', {'paths': ['templates/job-snippets'], 'files': [job + '.j2', 'generic.j2']}) | relpath(template_fullpath | dirname) %}{{ '' }} {% endfor %} {% if prometheus_server_jobs_extra is defined %} diff --git a/roles/monitoring/prometheus/server/templates/targets/blackbox-https.yml.j2 b/roles/monitoring/prometheus/server/templates/targets/blackbox-https.yml.j2 deleted file mode 100644 index b1a33df3..00000000 --- a/roles/monitoring/prometheus/server/templates/targets/blackbox-https.yml.j2 +++ /dev/null @@ -1,4 +0,0 @@ -- targets: [ '{{ hostvars[target.config.exporter_hostname].prometheus_scrape_endpoint }}' ] - labels: - instance: '{{ target.instance }}' - __param_target: {{ target.config.address }} diff --git a/roles/monitoring/prometheus/server/templates/targets/blackbox-ping.yml.j2 b/roles/monitoring/prometheus/server/templates/targets/blackbox-ping.yml.j2 deleted file mode 100644 index b1a33df3..00000000 --- a/roles/monitoring/prometheus/server/templates/targets/blackbox-ping.yml.j2 +++ /dev/null @@ -1,4 +0,0 @@ -- targets: [ '{{ hostvars[target.config.exporter_hostname].prometheus_scrape_endpoint }}' ] - labels: - instance: '{{ target.instance }}' - __param_target: {{ target.config.address }} diff --git a/roles/monitoring/prometheus/server/templates/targets/blackbox-ssh.yml.j2 b/roles/monitoring/prometheus/server/templates/targets/blackbox-ssh.yml.j2 deleted file mode 100644 index b1a33df3..00000000 --- a/roles/monitoring/prometheus/server/templates/targets/blackbox-ssh.yml.j2 +++ /dev/null @@ -1,4 +0,0 @@ -- targets: [ '{{ hostvars[target.config.exporter_hostname].prometheus_scrape_endpoint }}' ] - labels: - instance: '{{ target.instance }}' - __param_target: {{ target.config.address }} diff --git a/roles/monitoring/prometheus/server/templates/targets/blackbox/https.yml.j2 b/roles/monitoring/prometheus/server/templates/targets/blackbox/https.yml.j2 new file mode 100644 index 00000000..29c89590 --- /dev/null +++ b/roles/monitoring/prometheus/server/templates/targets/blackbox/https.yml.j2 @@ -0,0 +1,4 @@ +- targets: [ '{{ hostvars[target.config.exporter_hostname].prometheus_scrape_endpoint }}' ] + labels: + instance: '{{ target.instance }}' + __param_target: '{{ target.config.address }}' diff --git a/roles/monitoring/prometheus/server/templates/targets/blackbox/ping.yml.j2 b/roles/monitoring/prometheus/server/templates/targets/blackbox/ping.yml.j2 new file mode 100644 index 00000000..29c89590 --- /dev/null +++ b/roles/monitoring/prometheus/server/templates/targets/blackbox/ping.yml.j2 @@ -0,0 +1,4 @@ +- targets: [ '{{ hostvars[target.config.exporter_hostname].prometheus_scrape_endpoint }}' ] + labels: + instance: '{{ target.instance }}' + __param_target: '{{ target.config.address }}' diff --git a/roles/monitoring/prometheus/server/templates/targets/blackbox/ssh.yml.j2 b/roles/monitoring/prometheus/server/templates/targets/blackbox/ssh.yml.j2 new file mode 100644 index 00000000..29c89590 --- /dev/null +++ b/roles/monitoring/prometheus/server/templates/targets/blackbox/ssh.yml.j2 @@ -0,0 +1,4 @@ +- targets: [ '{{ hostvars[target.config.exporter_hostname].prometheus_scrape_endpoint }}' ] + labels: + instance: '{{ target.instance }}' + __param_target: '{{ target.config.address }}' diff --git a/roles/monitoring/prometheus/server/templates/targets/nut-ups.yml.j2 b/roles/monitoring/prometheus/server/templates/targets/nut-ups.yml.j2 deleted file mode 100644 index d63d79a7..00000000 --- a/roles/monitoring/prometheus/server/templates/targets/nut-ups.yml.j2 +++ /dev/null @@ -1,17 +0,0 @@ -- targets: [ '{{ hostvars[target.config.exporter_hostname].prometheus_scrape_endpoint }}' ] - labels: - instance: '{{ target.instance }}' - __param_ups: {{ target.config.ups }} - __param_server: {{ target.config.server | default('127.0.0.1') }} -{% if 'username' in target.config %} - __param_username: {{ target.config.username }} -{% endif %} -{% if 'password' in target.config %} - __param_password: {{ target.config.password }} -{% endif %} -{% if 'variables' in target.config %} - __param_variables: {{ target.config.variables }} -{% endif %} -{% if 'statuses' in target.config %} - __param_statuses: {{ target.config.statuses }} -{% endif %} diff --git a/roles/monitoring/prometheus/server/templates/targets/nut/ups.yml.j2 b/roles/monitoring/prometheus/server/templates/targets/nut/ups.yml.j2 new file mode 100644 index 00000000..6003cd46 --- /dev/null +++ b/roles/monitoring/prometheus/server/templates/targets/nut/ups.yml.j2 @@ -0,0 +1,17 @@ +- targets: [ '{{ hostvars[target.config.exporter_hostname].prometheus_scrape_endpoint }}' ] + labels: + instance: '{{ target.instance }}' + __param_ups: '{{ target.config.ups }}' + __param_server: '{{ target.config.server | default('127.0.0.1') }}' +{% if 'username' in target.config %} + __param_username: '{{ target.config.username }}' +{% endif %} +{% if 'password' in target.config %} + __param_password: '{{ target.config.password }}' +{% endif %} +{% if 'variables' in target.config %} + __param_variables: '{{ target.config.variables }}' +{% endif %} +{% if 'statuses' in target.config %} + __param_statuses: '{{ target.config.statuses }}' +{% endif %} -- cgit v1.2.3 From ec55b1572702b91184d99fec89fec537cfe2ea1f Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Thu, 23 Sep 2021 20:18:08 +0200 Subject: force blackbox exporter to ipv4 by default --- inventory/host_vars/ch-mon.yml | 7 +++++-- roles/monitoring/prometheus/exporter/blackbox/defaults/main.yml | 7 +++++++ 2 files changed, 12 insertions(+), 2 deletions(-) (limited to 'inventory') diff --git a/inventory/host_vars/ch-mon.yml b/inventory/host_vars/ch-mon.yml index 242c4835..d1a710b9 100644 --- a/inventory/host_vars/ch-mon.yml +++ b/inventory/host_vars/ch-mon.yml @@ -86,8 +86,11 @@ prometheus_job_blackbox__ping: prometheus_job_blackbox__https: - exporter_hostname: ch-mon - instance: "https-web.chaos-at-home.org" - address: web.chaos-at-home.org + instance: "https-pan.chaos-at-home.org" + address: "https://pan.chaos-at-home.org" + - exporter_hostname: ch-mon + instance: "https-mimas.chaos-at-home.org" + address: "https://mimas.chaos-at-home.org" prometheus_job_blackbox__ssh: - exporter_hostname: ch-mon diff --git a/roles/monitoring/prometheus/exporter/blackbox/defaults/main.yml b/roles/monitoring/prometheus/exporter/blackbox/defaults/main.yml index 4e7d8d9a..73b9fde1 100644 --- a/roles/monitoring/prometheus/exporter/blackbox/defaults/main.yml +++ b/roles/monitoring/prometheus/exporter/blackbox/defaults/main.yml @@ -2,23 +2,30 @@ prometheus_exporter_blackbox_modules: tcp_connect: prober: tcp + tcp: + preferred_ip_protocol: "ip4" tcp_tls_connect: prober: tcp tcp: + preferred_ip_protocol: "ip4" tls: true tls_config: insecure_skip_verify: true http_2xx: prober: http + http: + preferred_ip_protocol: "ip4" http_tls_2xx: prober: http http: + preferred_ip_protocol: "ip4" fail_if_not_ssl: true tls_config: insecure_skip_verify: true ssh_banner: prober: tcp tcp: + preferred_ip_protocol: "ip4" query_response: - expect: "^SSH-2.0-" - send: "SSH-2.0-blackbox-ssh-check" -- cgit v1.2.3 From 063bdb70a8e8353908ca9742e05be8fac65a61bf Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Sat, 25 Sep 2021 23:36:40 +0200 Subject: move away from exporter-exporter in favor for nginx --- chaos-at-home/ch-testvm-prometheus.yml | 1 + inventory/host_vars/ch-testvm-prometheus.yml | 3 ++ .../prometheus/exporter/base/defaults/main.yml | 2 +- .../prometheus/exporter/base/handlers/main.yml | 6 ++-- .../prometheus/exporter/base/tasks/main.yml | 30 +++++--------------- .../exporter/base/templates/nginx-vhost.j2 | 19 +++++++++++++ .../prometheus/exporter/base/templates/service.j2 | 32 ---------------------- .../prometheus/exporter/blackbox/handlers/main.yml | 7 ++--- .../prometheus/exporter/blackbox/tasks/main.yml | 11 ++++---- .../prometheus/exporter/ipmi/handlers/main.yml | 7 ++--- .../prometheus/exporter/ipmi/tasks/main.yml | 21 ++++++-------- roles/monitoring/prometheus/exporter/meta/main.yml | 10 ++++--- .../prometheus/exporter/mikrotik/handlers/main.yml | 7 ++--- .../prometheus/exporter/mikrotik/tasks/main.yml | 10 +++---- .../prometheus/exporter/node/handlers/main.yml | 7 ++--- .../prometheus/exporter/node/tasks/main.yml | 10 +++---- .../prometheus/exporter/nut/handlers/main.yml | 7 ++--- .../prometheus/exporter/nut/tasks/main.yml | 14 ++++++---- .../server/templates/jobs/blackbox/https.j2 | 3 +- .../server/templates/jobs/blackbox/ping.j2 | 3 +- .../server/templates/jobs/blackbox/ssh.j2 | 3 +- .../prometheus/server/templates/jobs/generic.j2 | 5 +--- .../prometheus/server/templates/jobs/node.j2 | 5 +--- .../prometheus/server/templates/jobs/nut/ups.j2 | 5 +--- 24 files changed, 92 insertions(+), 136 deletions(-) create mode 100644 roles/monitoring/prometheus/exporter/base/templates/nginx-vhost.j2 delete mode 100644 roles/monitoring/prometheus/exporter/base/templates/service.j2 (limited to 'inventory') diff --git a/chaos-at-home/ch-testvm-prometheus.yml b/chaos-at-home/ch-testvm-prometheus.yml index 3fd99d41..c0f33b8f 100644 --- a/chaos-at-home/ch-testvm-prometheus.yml +++ b/chaos-at-home/ch-testvm-prometheus.yml @@ -7,6 +7,7 @@ - role: core/sshd/base - role: core/zsh - role: core/ntp + - role: nginx/base - role: apt-repo/spreadspace - role: monitoring/prometheus/exporter # - role: kubernetes/base diff --git a/inventory/host_vars/ch-testvm-prometheus.yml b/inventory/host_vars/ch-testvm-prometheus.yml index e539735f..939fa398 100644 --- a/inventory/host_vars/ch-testvm-prometheus.yml +++ b/inventory/host_vars/ch-testvm-prometheus.yml @@ -36,6 +36,9 @@ network: spreadspace_apt_repo_components: - prometheus +prometheus_exporters_extra: + - ipmi + containerd_storage: type: lvm diff --git a/roles/monitoring/prometheus/exporter/base/defaults/main.yml b/roles/monitoring/prometheus/exporter/base/defaults/main.yml index 963763a5..613943d8 100644 --- a/roles/monitoring/prometheus/exporter/base/defaults/main.yml +++ b/roles/monitoring/prometheus/exporter/base/defaults/main.yml @@ -1,2 +1,2 @@ --- -prometheus_exporter_listen: ":9999" +prometheus_exporter_listen: "9999" diff --git a/roles/monitoring/prometheus/exporter/base/handlers/main.yml b/roles/monitoring/prometheus/exporter/base/handlers/main.yml index ebd760cf..d4e42ca0 100644 --- a/roles/monitoring/prometheus/exporter/base/handlers/main.yml +++ b/roles/monitoring/prometheus/exporter/base/handlers/main.yml @@ -1,5 +1,5 @@ --- -- name: restart prometheus-exporter-exporter +- name: reload nginx service: - name: prometheus-exporter-exporter - state: restarted + name: nginx + state: reloaded diff --git a/roles/monitoring/prometheus/exporter/base/tasks/main.yml b/roles/monitoring/prometheus/exporter/base/tasks/main.yml index eeb2a23d..5f42867d 100644 --- a/roles/monitoring/prometheus/exporter/base/tasks/main.yml +++ b/roles/monitoring/prometheus/exporter/base/tasks/main.yml @@ -6,17 +6,6 @@ - spreadspace_apt_repo_components is defined - "'prometheus' in spreadspace_apt_repo_components" - ## TODO: pin version -- name: install apt packages - apt: - name: prom-exporter-exporter - state: present - -- name: create configuration directories - file: - path: /etc/prometheus/exporter/exporter - state: directory - - name: add user for prometheus-exporter user: name: prometheus-exporter @@ -27,15 +16,10 @@ - name: create TLS certificate and key import_tasks: tls.yml -- name: generate systemd service unit - template: - src: service.j2 - dest: /etc/systemd/system/prometheus-exporter-exporter.service - notify: restart prometheus-exporter-exporter - -- name: make sure prometheus-exporter-exporter is enabled and started - systemd: - name: prometheus-exporter-exporter.service - daemon_reload: yes - state: started - enabled: yes +- name: configure nginx vhost + import_role: + name: nginx/vhost + vars: + nginx_vhost: + name: prometheus-exporter + content: "{{ lookup('template', 'nginx-vhost.j2') }}" diff --git a/roles/monitoring/prometheus/exporter/base/templates/nginx-vhost.j2 b/roles/monitoring/prometheus/exporter/base/templates/nginx-vhost.j2 new file mode 100644 index 00000000..70e65b29 --- /dev/null +++ b/roles/monitoring/prometheus/exporter/base/templates/nginx-vhost.j2 @@ -0,0 +1,19 @@ +server { + listen {{ prometheus_exporter_listen }} ssl; + server_name _; + + ssl_certificate /etc/ssl/prometheus/exporter/crt.pem; + ssl_certificate_key /etc/ssl/prometheus/exporter/key.pem; + ssl_client_certificate /etc/ssl/prometheus/ca-crt.pem; + ssl_verify_client on; + + root /nonexistent; + + location = / { + return 404 'please specify the exporter you want to reach!'; + } + + include snippets/proxy-nobuff.conf; + + include /etc/prometheus/exporter/*.locations; +} diff --git a/roles/monitoring/prometheus/exporter/base/templates/service.j2 b/roles/monitoring/prometheus/exporter/base/templates/service.j2 deleted file mode 100644 index 3d44744a..00000000 --- a/roles/monitoring/prometheus/exporter/base/templates/service.j2 +++ /dev/null @@ -1,32 +0,0 @@ -[Unit] -Description=Prometheus exporter proxy - -[Service] -Restart=always -User=prometheus-exporter -ExecStart=/usr/bin/prometheus-exporter-exporter -config.dirs=/etc/prometheus/exporter/exporter -config.file="" -web.listen-address="" -web.tls.listen-address="{{ prometheus_exporter_listen }}" -web.tls.cert="/etc/ssl/prometheus/exporter/crt.pem" -web.tls.key="/etc/ssl/prometheus/exporter/key.pem" --web.tls.ca="/etc/ssl/prometheus/ca-crt.pem" -web.tls.verify -{# TODO: implement reloading once the exporter_exporter supports this #} - -# systemd hardening-options -AmbientCapabilities= -CapabilityBoundingSet= -DeviceAllow=/dev/null rw -DevicePolicy=strict -LockPersonality=true -MemoryDenyWriteExecute=true -NoNewPrivileges=true -PrivateDevices=true -PrivateTmp=true -PrivateUsers=true -ProtectControlGroups=true -ProtectHome=true -ProtectKernelModules=true -ProtectKernelTunables=true -ProtectSystem=strict -RemoveIPC=true -RestrictNamespaces=true -RestrictRealtime=true -SystemCallArchitectures=native - -[Install] -WantedBy=multi-user.target diff --git a/roles/monitoring/prometheus/exporter/blackbox/handlers/main.yml b/roles/monitoring/prometheus/exporter/blackbox/handlers/main.yml index 99a416e2..12250769 100644 --- a/roles/monitoring/prometheus/exporter/blackbox/handlers/main.yml +++ b/roles/monitoring/prometheus/exporter/blackbox/handlers/main.yml @@ -9,8 +9,7 @@ name: prometheus-blackbox-exporter state: reloaded -- name: reload prometheus-exporter-exporter +- name: reload nginx service: - name: prometheus-exporter-exporter - ## TODO: implement reload once exporter_exporter supports this... - state: restarted + name: nginx + state: reloaded diff --git a/roles/monitoring/prometheus/exporter/blackbox/tasks/main.yml b/roles/monitoring/prometheus/exporter/blackbox/tasks/main.yml index 782c3561..f9793df6 100644 --- a/roles/monitoring/prometheus/exporter/blackbox/tasks/main.yml +++ b/roles/monitoring/prometheus/exporter/blackbox/tasks/main.yml @@ -32,9 +32,8 @@ - name: register exporter copy: content: | - method: http - http: - port: 9115 - path: /probe - dest: /etc/prometheus/exporter/exporter/blackbox.yml - notify: reload prometheus-exporter-exporter + location = /blackbox { + proxy_pass http://127.0.0.1:9115/probe; + } + dest: /etc/prometheus/exporter/blackbox.locations + notify: reload nginx diff --git a/roles/monitoring/prometheus/exporter/ipmi/handlers/main.yml b/roles/monitoring/prometheus/exporter/ipmi/handlers/main.yml index 40a945ae..a8eb55b3 100644 --- a/roles/monitoring/prometheus/exporter/ipmi/handlers/main.yml +++ b/roles/monitoring/prometheus/exporter/ipmi/handlers/main.yml @@ -9,8 +9,7 @@ name: prometheus-ipmi-exporter state: reloaded -- name: reload prometheus-exporter-exporter +- name: reload nginx service: - name: prometheus-exporter-exporter - ## TODO: implement reload once exporter_exporter supports this... - state: restarted + name: nginx + state: reloaded diff --git a/roles/monitoring/prometheus/exporter/ipmi/tasks/main.yml b/roles/monitoring/prometheus/exporter/ipmi/tasks/main.yml index 9e63f692..91318f16 100644 --- a/roles/monitoring/prometheus/exporter/ipmi/tasks/main.yml +++ b/roles/monitoring/prometheus/exporter/ipmi/tasks/main.yml @@ -30,18 +30,13 @@ enabled: yes - name: register exporter - loop: - - name: local - path: /metrics - - name: remote - path: /ipmi - loop_control: - label: "{{ item.name }}" copy: content: | - method: http - http: - port: 9290 - path: {{ item.path }} - dest: "/etc/prometheus/exporter/exporter/ipmi-{{ item.name }}.yml" - notify: reload prometheus-exporter-exporter + location = /ipmi { + proxy_pass http://127.0.0.1:9290/metrics; + } + location = /ipmi/remote { + proxy_pass http://127.0.0.1:9290/ipmi; + } + dest: /etc/prometheus/exporter/ipmi.locations + notify: reload nginx diff --git a/roles/monitoring/prometheus/exporter/meta/main.yml b/roles/monitoring/prometheus/exporter/meta/main.yml index 22131422..68fce6cb 100644 --- a/roles/monitoring/prometheus/exporter/meta/main.yml +++ b/roles/monitoring/prometheus/exporter/meta/main.yml @@ -1,11 +1,13 @@ --- dependencies: - role: monitoring/prometheus/exporter/base - - role: monitoring/prometheus/exporter/node - when: "'node' in (prometheus_exporters_default | union(prometheus_exporters_extra))" - role: monitoring/prometheus/exporter/blackbox when: "'blackbox' in (prometheus_exporters_default | union(prometheus_exporters_extra))" - - role: monitoring/prometheus/exporter/nut - when: "'nut' in (prometheus_exporters_default | union(prometheus_exporters_extra))" + - role: monitoring/prometheus/exporter/ipmi + when: "'ipmi' in (prometheus_exporters_default | union(prometheus_exporters_extra))" - role: monitoring/prometheus/exporter/mikrotik when: "'mikrotik' in (prometheus_exporters_default | union(prometheus_exporters_extra))" + - role: monitoring/prometheus/exporter/node + when: "'node' in (prometheus_exporters_default | union(prometheus_exporters_extra))" + - role: monitoring/prometheus/exporter/nut + when: "'nut' in (prometheus_exporters_default | union(prometheus_exporters_extra))" diff --git a/roles/monitoring/prometheus/exporter/mikrotik/handlers/main.yml b/roles/monitoring/prometheus/exporter/mikrotik/handlers/main.yml index cb85d0d9..c5844220 100644 --- a/roles/monitoring/prometheus/exporter/mikrotik/handlers/main.yml +++ b/roles/monitoring/prometheus/exporter/mikrotik/handlers/main.yml @@ -4,8 +4,7 @@ name: prometheus-mikrotik-exporter state: restarted -- name: reload prometheus-exporter-exporter +- name: reload nginx service: - name: prometheus-exporter-exporter - ## TODO: implement reload once exporter_exporter supports this... - state: restarted + name: nginx + state: reloaded diff --git a/roles/monitoring/prometheus/exporter/mikrotik/tasks/main.yml b/roles/monitoring/prometheus/exporter/mikrotik/tasks/main.yml index 07219c68..72c78e4a 100644 --- a/roles/monitoring/prometheus/exporter/mikrotik/tasks/main.yml +++ b/roles/monitoring/prometheus/exporter/mikrotik/tasks/main.yml @@ -35,8 +35,8 @@ - name: register exporter copy: content: | - method: http - http: - port: 9436 - dest: /etc/prometheus/exporter/exporter/mikrotik.yml - notify: reload prometheus-exporter-exporter + location = /mikrotik { + proxy_pass http://127.0.0.1:9436/metrics; + } + dest: /etc/prometheus/exporter/mikrotik.locations + notify: reload nginx diff --git a/roles/monitoring/prometheus/exporter/node/handlers/main.yml b/roles/monitoring/prometheus/exporter/node/handlers/main.yml index 3e1b2000..56056ea6 100644 --- a/roles/monitoring/prometheus/exporter/node/handlers/main.yml +++ b/roles/monitoring/prometheus/exporter/node/handlers/main.yml @@ -4,8 +4,7 @@ name: prometheus-node-exporter state: restarted -- name: reload prometheus-exporter-exporter +- name: reload nginx service: - name: prometheus-exporter-exporter - ## TODO: implement reload once exporter_exporter supports this... - state: restarted + name: nginx + state: reloaded diff --git a/roles/monitoring/prometheus/exporter/node/tasks/main.yml b/roles/monitoring/prometheus/exporter/node/tasks/main.yml index 56903a33..2811c759 100644 --- a/roles/monitoring/prometheus/exporter/node/tasks/main.yml +++ b/roles/monitoring/prometheus/exporter/node/tasks/main.yml @@ -28,11 +28,11 @@ - name: register exporter copy: content: | - method: http - http: - port: 9100 - dest: /etc/prometheus/exporter/exporter/node.yml - notify: reload prometheus-exporter-exporter + location = /node { + proxy_pass http://127.0.0.1:9100/metrics; + } + dest: /etc/prometheus/exporter/node.locations + notify: reload nginx - name: create directory for textfile collector scripts file: diff --git a/roles/monitoring/prometheus/exporter/nut/handlers/main.yml b/roles/monitoring/prometheus/exporter/nut/handlers/main.yml index 6e10f43b..edd87ed5 100644 --- a/roles/monitoring/prometheus/exporter/nut/handlers/main.yml +++ b/roles/monitoring/prometheus/exporter/nut/handlers/main.yml @@ -4,8 +4,7 @@ name: prometheus-nut-exporter state: restarted -- name: reload prometheus-exporter-exporter +- name: reload ngnix service: - name: prometheus-exporter-exporter - ## TODO: implement reload once exporter_exporter supports this... - state: restarted + name: nginx + state: reloaded diff --git a/roles/monitoring/prometheus/exporter/nut/tasks/main.yml b/roles/monitoring/prometheus/exporter/nut/tasks/main.yml index 8245feae..f602472d 100644 --- a/roles/monitoring/prometheus/exporter/nut/tasks/main.yml +++ b/roles/monitoring/prometheus/exporter/nut/tasks/main.yml @@ -21,9 +21,11 @@ - name: register exporter copy: content: | - method: http - http: - port: 9199 - path: /ups_metrics - dest: /etc/prometheus/exporter/exporter/nut.yml - notify: reload prometheus-exporter-exporter + location = /nut { + proxy_pass http://127.0.0.1:9199/metrics; + } + location = /nut/ups { + proxy_pass http://127.0.0.1:9199/ups_metrics; + } + dest: /etc/prometheus/exporter/nut.locations + notify: reload nginx diff --git a/roles/monitoring/prometheus/server/templates/jobs/blackbox/https.j2 b/roles/monitoring/prometheus/server/templates/jobs/blackbox/https.j2 index 98a64121..86ff88dd 100644 --- a/roles/monitoring/prometheus/server/templates/jobs/blackbox/https.j2 +++ b/roles/monitoring/prometheus/server/templates/jobs/blackbox/https.j2 @@ -1,8 +1,7 @@ - job_name: '{{ job }}' - metrics_path: /proxy + metrics_path: /blackbox params: module: - - blackbox - http_tls_2xx scheme: https tls_config: diff --git a/roles/monitoring/prometheus/server/templates/jobs/blackbox/ping.j2 b/roles/monitoring/prometheus/server/templates/jobs/blackbox/ping.j2 index 736ffec1..2d3889d2 100644 --- a/roles/monitoring/prometheus/server/templates/jobs/blackbox/ping.j2 +++ b/roles/monitoring/prometheus/server/templates/jobs/blackbox/ping.j2 @@ -1,8 +1,7 @@ - job_name: '{{ job }}' - metrics_path: /proxy + metrics_path: /blackbox params: module: - - blackbox - icmp scheme: https tls_config: diff --git a/roles/monitoring/prometheus/server/templates/jobs/blackbox/ssh.j2 b/roles/monitoring/prometheus/server/templates/jobs/blackbox/ssh.j2 index 166f37ad..97565673 100644 --- a/roles/monitoring/prometheus/server/templates/jobs/blackbox/ssh.j2 +++ b/roles/monitoring/prometheus/server/templates/jobs/blackbox/ssh.j2 @@ -1,8 +1,7 @@ - job_name: '{{ job }}' - metrics_path: /proxy + metrics_path: /blackbox params: module: - - blackbox - ssh_banner scheme: https tls_config: diff --git a/roles/monitoring/prometheus/server/templates/jobs/generic.j2 b/roles/monitoring/prometheus/server/templates/jobs/generic.j2 index b155c5f7..65a95007 100644 --- a/roles/monitoring/prometheus/server/templates/jobs/generic.j2 +++ b/roles/monitoring/prometheus/server/templates/jobs/generic.j2 @@ -1,8 +1,5 @@ - job_name: '{{ job }}' - metrics_path: /proxy - params: - module: - - {{ job }} + metrics_path: /{{ job }} scheme: https tls_config: ca_file: /etc/ssl/prometheus/ca-crt.pem diff --git a/roles/monitoring/prometheus/server/templates/jobs/node.j2 b/roles/monitoring/prometheus/server/templates/jobs/node.j2 index ba9eab31..1b14e1f6 100644 --- a/roles/monitoring/prometheus/server/templates/jobs/node.j2 +++ b/roles/monitoring/prometheus/server/templates/jobs/node.j2 @@ -1,8 +1,5 @@ - job_name: '{{ job }}' - metrics_path: /proxy - params: - module: - - {{ job }} + metrics_path: /{{ job }} scheme: https tls_config: ca_file: /etc/ssl/prometheus/ca-crt.pem diff --git a/roles/monitoring/prometheus/server/templates/jobs/nut/ups.j2 b/roles/monitoring/prometheus/server/templates/jobs/nut/ups.j2 index 3a2c5c62..0cf4ae4e 100644 --- a/roles/monitoring/prometheus/server/templates/jobs/nut/ups.j2 +++ b/roles/monitoring/prometheus/server/templates/jobs/nut/ups.j2 @@ -1,8 +1,5 @@ - job_name: '{{ job }}' - metrics_path: /proxy - params: - module: - - nut + metrics_path: /nut/ups scheme: https tls_config: ca_file: /etc/ssl/prometheus/ca-crt.pem -- cgit v1.2.3 From cc89d6d4211aa5aec8e5bef8c854d4929c337887 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Sun, 26 Sep 2021 03:32:47 +0200 Subject: improved promethues multitarget support --- inventory/group_vars/chaos-at-home-ups/vars.yml | 6 +- inventory/group_vars/ele-ups/vars.yml | 10 +-- .../group_vars/promzone-chaos-at-home/vars.yml | 6 +- .../group_vars/promzone-elevate-festival/vars.yml | 1 + inventory/host_vars/ch-mon.yml | 39 ++++++------ .../prometheus/exporter/blackbox/tasks/main.yml | 3 + .../prometheus/server/defaults/main/main.yml | 4 +- .../server/defaults/main/rules_blackbox.yml | 46 +------------- .../server/defaults/main/rules_blackbox__https.yml | 20 ------ .../server/defaults/main/rules_blackbox__ping.yml | 11 ---- .../server/defaults/main/rules_blackbox__probe.yml | 74 ++++++++++++++++++++++ .../server/defaults/main/rules_blackbox__ssh.yml | 3 - .../prometheus/server/filter_plugins/prometheus.py | 10 +-- roles/monitoring/prometheus/server/tasks/main.yml | 2 +- .../server/templates/jobs/blackbox/https.j2 | 13 ---- .../server/templates/jobs/blackbox/ping.j2 | 13 ---- .../server/templates/jobs/blackbox/ssh.j2 | 13 ---- .../prometheus/server/templates/jobs/nut/ups.j2 | 10 --- .../server/templates/targets/blackbox/https.yml.j2 | 4 -- .../server/templates/targets/blackbox/ping.yml.j2 | 4 -- .../server/templates/targets/blackbox/probe.yml.j2 | 5 ++ .../server/templates/targets/blackbox/ssh.yml.j2 | 4 -- .../server/templates/targets/nut/ups.yml.j2 | 2 +- 23 files changed, 123 insertions(+), 180 deletions(-) delete mode 100644 roles/monitoring/prometheus/server/defaults/main/rules_blackbox__https.yml delete mode 100644 roles/monitoring/prometheus/server/defaults/main/rules_blackbox__ping.yml create mode 100644 roles/monitoring/prometheus/server/defaults/main/rules_blackbox__probe.yml delete mode 100644 roles/monitoring/prometheus/server/defaults/main/rules_blackbox__ssh.yml delete mode 100644 roles/monitoring/prometheus/server/templates/jobs/blackbox/https.j2 delete mode 100644 roles/monitoring/prometheus/server/templates/jobs/blackbox/ping.j2 delete mode 100644 roles/monitoring/prometheus/server/templates/jobs/blackbox/ssh.j2 delete mode 100644 roles/monitoring/prometheus/server/templates/jobs/nut/ups.j2 delete mode 100644 roles/monitoring/prometheus/server/templates/targets/blackbox/https.yml.j2 delete mode 100644 roles/monitoring/prometheus/server/templates/targets/blackbox/ping.yml.j2 create mode 100644 roles/monitoring/prometheus/server/templates/targets/blackbox/probe.yml.j2 delete mode 100644 roles/monitoring/prometheus/server/templates/targets/blackbox/ssh.yml.j2 (limited to 'inventory') diff --git a/inventory/group_vars/chaos-at-home-ups/vars.yml b/inventory/group_vars/chaos-at-home-ups/vars.yml index f8c1bdf1..7b60e893 100644 --- a/inventory/group_vars/chaos-at-home-ups/vars.yml +++ b/inventory/group_vars/chaos-at-home-ups/vars.yml @@ -11,8 +11,8 @@ prometheus_scrape_endpoint: "{{ network_mgmt_zone.prefix | ipaddr(network_mgmt_z prometheus_exporters_default: - openwrt -prometheus_job_nut__ups: - - exporter_hostname: ch-mon - instance: "ups-{{ ups_name }}" +prometheus_job_multitarget_nut__ups: + ch-mon: + - instance: "ups-{{ ups_name }}" ups: "{{ ups_name }}" server: "{{ network_mgmt_zone.prefix | ipaddr(network_mgmt_zone.offsets[inventory_hostname]) | ipaddr('address') }}" diff --git a/inventory/group_vars/ele-ups/vars.yml b/inventory/group_vars/ele-ups/vars.yml index 1c4613a3..28a5eaff 100644 --- a/inventory/group_vars/ele-ups/vars.yml +++ b/inventory/group_vars/ele-ups/vars.yml @@ -14,8 +14,8 @@ prometheus_scrape_endpoint: "{{ network_mgmt_zone.prefix | ipaddr(network_mgmt_z prometheus_exporters_default: - openwrt -prometheus_job_nut__ups: - exporter_hostname: ele-mon - instance: "ups-{{ ups_name }}" - ups: "{{ ups_name }}" - server: "{{ network_mgmt_zone.prefix | ipaddr(network_mgmt_zone.offsets[inventory_hostname]) | ipaddr('address') }}" +prometheus_job_multitarget_nut__ups: + ele-mon: + - instance: "ups-{{ ups_name }}" + ups: "{{ ups_name }}" + server: "{{ network_mgmt_zone.prefix | ipaddr(network_mgmt_zone.offsets[inventory_hostname]) | ipaddr('address') }}" diff --git a/inventory/group_vars/promzone-chaos-at-home/vars.yml b/inventory/group_vars/promzone-chaos-at-home/vars.yml index 84ed1263..529bf3e7 100644 --- a/inventory/group_vars/promzone-chaos-at-home/vars.yml +++ b/inventory/group_vars/promzone-chaos-at-home/vars.yml @@ -9,10 +9,10 @@ prometheus_server: ch-mon prometheus_server_jobs: - node - openwrt + - nut - nut/ups - - blackbox/ping - - blackbox/https - - blackbox/ssh + - blackbox + - blackbox/probe prometheus_zone_name: chaos@home prometheus_zone_targets: "{{ groups['promzone-chaos-at-home'] }}" diff --git a/inventory/group_vars/promzone-elevate-festival/vars.yml b/inventory/group_vars/promzone-elevate-festival/vars.yml index 43115dc4..b3321614 100644 --- a/inventory/group_vars/promzone-elevate-festival/vars.yml +++ b/inventory/group_vars/promzone-elevate-festival/vars.yml @@ -9,6 +9,7 @@ prometheus_server: ele-mon prometheus_server_jobs: - node - openwrt + - nut - nut/ups prometheus_zone_name: Elevate Festival diff --git a/inventory/host_vars/ch-mon.yml b/inventory/host_vars/ch-mon.yml index d1a710b9..b2402d0c 100644 --- a/inventory/host_vars/ch-mon.yml +++ b/inventory/host_vars/ch-mon.yml @@ -76,26 +76,25 @@ prometheus_exporter_blackbox_modules_extra: icmp: prober: icmp -prometheus_job_blackbox__ping: - - exporter_hostname: ch-mon - instance: "ping-magentagw" - address: 62.99.185.129 - - exporter_hostname: ch-mon - instance: "ping-quad9" - address: 9.9.9.9 - -prometheus_job_blackbox__https: - - exporter_hostname: ch-mon - instance: "https-pan.chaos-at-home.org" - address: "https://pan.chaos-at-home.org" - - exporter_hostname: ch-mon - instance: "https-mimas.chaos-at-home.org" - address: "https://mimas.chaos-at-home.org" - -prometheus_job_blackbox__ssh: - - exporter_hostname: ch-mon - instance: "ssh-{{ inventory_hostname }}" - address: "{{ network_zones.svc.prefix | ipaddr(network_zones.svc.offsets[inventory_hostname]) | ipaddr('address') }}:{{ ansible_port | default(22) }}" +prometheus_job_multitarget_blackbox__probe: + ch-mon: + - instance: "ping-magentagw" + target: 62.99.185.129 + module: icmp + - instance: "ping-quad9" + target: 9.9.9.9 + module: icmp + + - instance: "https-pan.chaos-at-home.org" + target: "https://pan.chaos-at-home.org" + module: http_tls_2xx + - instance: "https-mimas.chaos-at-home.org" + target: "https://mimas.chaos-at-home.org" + module: http_tls_2xx + + - instance: "ssh-{{ inventory_hostname }}" + target: "{{ network_zones.svc.prefix | ipaddr(network_zones.svc.offsets[inventory_hostname]) | ipaddr('address') }}:{{ ansible_port | default(22) }}" + module: ssh_banner promethues_alertmanager_smtp: diff --git a/roles/monitoring/prometheus/exporter/blackbox/tasks/main.yml b/roles/monitoring/prometheus/exporter/blackbox/tasks/main.yml index f9793df6..c4cabfce 100644 --- a/roles/monitoring/prometheus/exporter/blackbox/tasks/main.yml +++ b/roles/monitoring/prometheus/exporter/blackbox/tasks/main.yml @@ -33,6 +33,9 @@ copy: content: | location = /blackbox { + proxy_pass http://127.0.0.1:9115/metrics; + } + location = /blackbox/probe { proxy_pass http://127.0.0.1:9115/probe; } dest: /etc/prometheus/exporter/blackbox.locations diff --git a/roles/monitoring/prometheus/server/defaults/main/main.yml b/roles/monitoring/prometheus/server/defaults/main/main.yml index 09cd150c..7781fd69 100644 --- a/roles/monitoring/prometheus/server/defaults/main/main.yml +++ b/roles/monitoring/prometheus/server/defaults/main/main.yml @@ -17,9 +17,7 @@ prometheus_server_rules: nut: "{{ prometheus_server_rules_nut + prometheus_server_rules_nut_extra }}" nut/ups: "{{ prometheus_server_rules_nut__ups + prometheus_server_rules_nut__ups_extra }}" blackbox: "{{ prometheus_server_rules_blackbox + prometheus_server_rules_blackbox_extra }}" - blackbox/ping: "{{ prometheus_server_rules_blackbox__ping + prometheus_server_rules_blackbox__ping_extra }}" - blackbox/https: "{{ prometheus_server_rules_blackbox__https + prometheus_server_rules_blackbox__https_extra }}" - blackbox/ssh: "{{ prometheus_server_rules_blackbox__ssh + prometheus_server_rules_blackbox__ssh_extra }}" + blackbox/probe: "{{ prometheus_server_rules_blackbox__probe + prometheus_server_rules_blackbox__probe_extra }}" # prometheus_server_alertmanager: # url: "127.0.0.1:9093" diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_blackbox.yml b/roles/monitoring/prometheus/server/defaults/main/rules_blackbox.yml index d5c1fd42..99f2e83c 100644 --- a/roles/monitoring/prometheus/server/defaults/main/rules_blackbox.yml +++ b/roles/monitoring/prometheus/server/defaults/main/rules_blackbox.yml @@ -1,47 +1,3 @@ --- prometheus_server_rules_blackbox_extra: [] -prometheus_server_rules_blackbox: - - alert: BlackboxProbeFailed - expr: probe_success == 0 - for: 0m - labels: - severity: critical - annotations: - summary: Blackbox probe failed (instance {{ '{{' }} $labels.instance {{ '}}' }}) - description: "Probe failed\n VALUE = {{ '{{' }} $value {{ '}}' }}\n LABELS = {{ '{{' }} $labels {{ '}}' }}" - - - alert: BlackboxSlowProbe - expr: avg_over_time(probe_duration_seconds[1m]) > 1 - for: 1m - labels: - severity: warning - annotations: - summary: Blackbox slow probe (instance {{ '{{' }} $labels.instance {{ '}}' }}) - description: "Blackbox probe took more than 1s to complete\n VALUE = {{ '{{' }} $value {{ '}}' }}\n LABELS = {{ '{{' }} $labels {{ '}}' }}" - - - alert: BlackboxSslCertificateWillExpireSoon - expr: probe_ssl_earliest_cert_expiry - time() < 86400 * 30 - for: 0m - labels: - severity: warning - annotations: - summary: Blackbox SSL certificate will expire soon (instance {{ '{{' }} $labels.instance {{ '}}' }}) - description: "SSL certificate expires in 30 days\n VALUE = {{ '{{' }} $value {{ '}}' }}\n LABELS = {{ '{{' }} $labels {{ '}}' }}" - - - alert: BlackboxSslCertificateWillExpireSoon - expr: probe_ssl_earliest_cert_expiry - time() < 86400 * 3 - for: 0m - labels: - severity: critical - annotations: - summary: Blackbox SSL certificate will expire soon (instance {{ '{{' }} $labels.instance {{ '}}' }}) - description: "SSL certificate expires in 3 days\n VALUE = {{ '{{' }} $value {{ '}}' }}\n LABELS = {{ '{{' }} $labels {{ '}}' }}" - - - alert: BlackboxSslCertificateExpired - expr: probe_ssl_earliest_cert_expiry - time() <= 0 - for: 0m - labels: - severity: critical - annotations: - summary: Blackbox SSL certificate expired (instance {{ '{{' }} $labels.instance {{ '}}' }}) - description: "SSL certificate has expired already\n VALUE = {{ '{{' }} $value {{ '}}' }}\n LABELS = {{ '{{' }} $labels {{ '}}' }}" +prometheus_server_rules_blackbox: [] diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_blackbox__https.yml b/roles/monitoring/prometheus/server/defaults/main/rules_blackbox__https.yml deleted file mode 100644 index 140e3b4f..00000000 --- a/roles/monitoring/prometheus/server/defaults/main/rules_blackbox__https.yml +++ /dev/null @@ -1,20 +0,0 @@ ---- -prometheus_server_rules_blackbox__https_extra: [] -prometheus_server_rules_blackbox__https: - - alert: BlackboxProbeHttpFailure - expr: probe_http_status_code <= 199 OR probe_http_status_code >= 400 - for: 0m - labels: - severity: critical - annotations: - summary: Blackbox probe HTTP failure (instance {{ '{{' }} $labels.instance {{ '}}' }}) - description: "HTTP status code is not 200-399\n VALUE = {{ '{{' }} $value {{ '}}' }}\n LABELS = {{ '{{' }} $labels {{ '}}' }}" - - - alert: BlackboxProbeSlowHttp - expr: avg_over_time(probe_http_duration_seconds[1m]) > 1 - for: 1m - labels: - severity: warning - annotations: - summary: Blackbox probe slow HTTP (instance {{ '{{' }} $labels.instance {{ '}}' }}) - description: "HTTP request took more than 1s\n VALUE = {{ '{{' }} $value {{ '}}' }}\n LABELS = {{ '{{' }} $labels {{ '}}' }}" diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_blackbox__ping.yml b/roles/monitoring/prometheus/server/defaults/main/rules_blackbox__ping.yml deleted file mode 100644 index cc87b6b1..00000000 --- a/roles/monitoring/prometheus/server/defaults/main/rules_blackbox__ping.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- -prometheus_server_rules_blackbox__ping_extra: [] -prometheus_server_rules_blackbox__ping: - - alert: BlackboxProbeSlowPing - expr: avg_over_time(probe_icmp_duration_seconds[1m]) > 1 - for: 1m - labels: - severity: warning - annotations: - summary: Blackbox probe slow ping (instance {{ '{{' }} $labels.instance {{ '}}' }}) - description: "Blackbox ping took more than 1s\n VALUE = {{ '{{' }} $value {{ '}}' }}\n LABELS = {{ '{{' }} $labels {{ '}}' }}" diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_blackbox__probe.yml b/roles/monitoring/prometheus/server/defaults/main/rules_blackbox__probe.yml new file mode 100644 index 00000000..9f9d2292 --- /dev/null +++ b/roles/monitoring/prometheus/server/defaults/main/rules_blackbox__probe.yml @@ -0,0 +1,74 @@ +--- +prometheus_server_rules_blackbox__probe_extra: [] +prometheus_server_rules_blackbox__probe: + - alert: BlackboxProbeFailed + expr: probe_success == 0 + for: 0m + labels: + severity: critical + annotations: + summary: Blackbox probe failed (instance {{ '{{' }} $labels.instance {{ '}}' }}) + description: "Probe failed\n VALUE = {{ '{{' }} $value {{ '}}' }}\n LABELS = {{ '{{' }} $labels {{ '}}' }}" + + - alert: BlackboxSlowProbe + expr: avg_over_time(probe_duration_seconds[1m]) > 1 + for: 1m + labels: + severity: warning + annotations: + summary: Blackbox slow probe (instance {{ '{{' }} $labels.instance {{ '}}' }}) + description: "Blackbox probe took more than 1s to complete\n VALUE = {{ '{{' }} $value {{ '}}' }}\n LABELS = {{ '{{' }} $labels {{ '}}' }}" + + - alert: BlackboxSslCertificateWillExpireSoon + expr: probe_ssl_earliest_cert_expiry - time() < 86400 * 30 + for: 0m + labels: + severity: warning + annotations: + summary: Blackbox SSL certificate will expire soon (instance {{ '{{' }} $labels.instance {{ '}}' }}) + description: "SSL certificate expires in 30 days\n VALUE = {{ '{{' }} $value {{ '}}' }}\n LABELS = {{ '{{' }} $labels {{ '}}' }}" + + - alert: BlackboxSslCertificateWillExpireSoon + expr: probe_ssl_earliest_cert_expiry - time() < 86400 * 3 + for: 0m + labels: + severity: critical + annotations: + summary: Blackbox SSL certificate will expire soon (instance {{ '{{' }} $labels.instance {{ '}}' }}) + description: "SSL certificate expires in 3 days\n VALUE = {{ '{{' }} $value {{ '}}' }}\n LABELS = {{ '{{' }} $labels {{ '}}' }}" + + - alert: BlackboxSslCertificateExpired + expr: probe_ssl_earliest_cert_expiry - time() <= 0 + for: 0m + labels: + severity: critical + annotations: + summary: Blackbox SSL certificate expired (instance {{ '{{' }} $labels.instance {{ '}}' }}) + description: "SSL certificate has expired already\n VALUE = {{ '{{' }} $value {{ '}}' }}\n LABELS = {{ '{{' }} $labels {{ '}}' }}" + + - alert: BlackboxProbeHttpFailure + expr: probe_http_status_code <= 199 OR probe_http_status_code >= 400 + for: 0m + labels: + severity: critical + annotations: + summary: Blackbox probe HTTP failure (instance {{ '{{' }} $labels.instance {{ '}}' }}) + description: "HTTP status code is not 200-399\n VALUE = {{ '{{' }} $value {{ '}}' }}\n LABELS = {{ '{{' }} $labels {{ '}}' }}" + + - alert: BlackboxProbeSlowHttp + expr: avg_over_time(probe_http_duration_seconds[1m]) > 1 + for: 1m + labels: + severity: warning + annotations: + summary: Blackbox probe slow HTTP (instance {{ '{{' }} $labels.instance {{ '}}' }}) + description: "HTTP request took more than 1s\n VALUE = {{ '{{' }} $value {{ '}}' }}\n LABELS = {{ '{{' }} $labels {{ '}}' }}" + + - alert: BlackboxProbeSlowPing + expr: avg_over_time(probe_icmp_duration_seconds[1m]) > 1 + for: 1m + labels: + severity: warning + annotations: + summary: Blackbox probe slow ping (instance {{ '{{' }} $labels.instance {{ '}}' }}) + description: "Blackbox ping took more than 1s\n VALUE = {{ '{{' }} $value {{ '}}' }}\n LABELS = {{ '{{' }} $labels {{ '}}' }}" diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_blackbox__ssh.yml b/roles/monitoring/prometheus/server/defaults/main/rules_blackbox__ssh.yml deleted file mode 100644 index 8e717c41..00000000 --- a/roles/monitoring/prometheus/server/defaults/main/rules_blackbox__ssh.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -prometheus_server_rules_blackbox__ssh_extra: [] -prometheus_server_rules_blackbox__ssh: [] diff --git a/roles/monitoring/prometheus/server/filter_plugins/prometheus.py b/roles/monitoring/prometheus/server/filter_plugins/prometheus.py index 1443e837..d91ef619 100644 --- a/roles/monitoring/prometheus/server/filter_plugins/prometheus.py +++ b/roles/monitoring/prometheus/server/filter_plugins/prometheus.py @@ -11,10 +11,12 @@ def prometheus_job_targets(hostvars, jobs, targets): result = [] for job in jobs: for target in targets: - special_config_varname = 'prometheus_job_' + job.replace('-', '_').replace('/', '__') - if special_config_varname in hostvars[target]: - for config in hostvars[target][special_config_varname]: - result.append({'job': job, 'instance': config['instance'], 'config': config, 'enabled': True}) + multitarget_config_varname = 'prometheus_job_multitarget_' + job.replace('-', '_').replace('/', '__') + if multitarget_config_varname in hostvars[target]: + for exporter_hostname, configs in hostvars[target][multitarget_config_varname].items(): + for config in configs: + result.append({'job': job, 'instance': config['instance'], 'enabled': True, + 'exporter_hostname': exporter_hostname, 'config': config}) else: enabled = job in hostvars[target]['prometheus_exporters_default'] or job in hostvars[target]['prometheus_exporters_extra'] diff --git a/roles/monitoring/prometheus/server/tasks/main.yml b/roles/monitoring/prometheus/server/tasks/main.yml index c0928cc3..16167c9c 100644 --- a/roles/monitoring/prometheus/server/tasks/main.yml +++ b/roles/monitoring/prometheus/server/tasks/main.yml @@ -83,7 +83,7 @@ state: directory - name: generate rules files for all jobs - loop: "{{ prometheus_server_jobs | union(['prometheus']) | union(prometheus_server_jobs | select('match', '.*/.*') | map('dirname') | unique) }}" + loop: "{{ prometheus_server_jobs | union(['prometheus']) }}" template: src: rules.yml.j2 dest: "/etc/prometheus/rules/{{ item }}.yml" diff --git a/roles/monitoring/prometheus/server/templates/jobs/blackbox/https.j2 b/roles/monitoring/prometheus/server/templates/jobs/blackbox/https.j2 deleted file mode 100644 index 86ff88dd..00000000 --- a/roles/monitoring/prometheus/server/templates/jobs/blackbox/https.j2 +++ /dev/null @@ -1,13 +0,0 @@ - - job_name: '{{ job }}' - metrics_path: /blackbox - params: - module: - - http_tls_2xx - scheme: https - tls_config: - ca_file: /etc/ssl/prometheus/ca-crt.pem - cert_file: /etc/ssl/prometheus/server/scrape-crt.pem - key_file: /etc/ssl/prometheus/server/scrape-key.pem - file_sd_configs: - - files: - - "/etc/prometheus/targets/{{ job }}/*.yml" diff --git a/roles/monitoring/prometheus/server/templates/jobs/blackbox/ping.j2 b/roles/monitoring/prometheus/server/templates/jobs/blackbox/ping.j2 deleted file mode 100644 index 2d3889d2..00000000 --- a/roles/monitoring/prometheus/server/templates/jobs/blackbox/ping.j2 +++ /dev/null @@ -1,13 +0,0 @@ - - job_name: '{{ job }}' - metrics_path: /blackbox - params: - module: - - icmp - scheme: https - tls_config: - ca_file: /etc/ssl/prometheus/ca-crt.pem - cert_file: /etc/ssl/prometheus/server/scrape-crt.pem - key_file: /etc/ssl/prometheus/server/scrape-key.pem - file_sd_configs: - - files: - - "/etc/prometheus/targets/{{ job }}/*.yml" diff --git a/roles/monitoring/prometheus/server/templates/jobs/blackbox/ssh.j2 b/roles/monitoring/prometheus/server/templates/jobs/blackbox/ssh.j2 deleted file mode 100644 index 97565673..00000000 --- a/roles/monitoring/prometheus/server/templates/jobs/blackbox/ssh.j2 +++ /dev/null @@ -1,13 +0,0 @@ - - job_name: '{{ job }}' - metrics_path: /blackbox - params: - module: - - ssh_banner - scheme: https - tls_config: - ca_file: /etc/ssl/prometheus/ca-crt.pem - cert_file: /etc/ssl/prometheus/server/scrape-crt.pem - key_file: /etc/ssl/prometheus/server/scrape-key.pem - file_sd_configs: - - files: - - "/etc/prometheus/targets/{{ job }}/*.yml" diff --git a/roles/monitoring/prometheus/server/templates/jobs/nut/ups.j2 b/roles/monitoring/prometheus/server/templates/jobs/nut/ups.j2 deleted file mode 100644 index 0cf4ae4e..00000000 --- a/roles/monitoring/prometheus/server/templates/jobs/nut/ups.j2 +++ /dev/null @@ -1,10 +0,0 @@ - - job_name: '{{ job }}' - metrics_path: /nut/ups - scheme: https - tls_config: - ca_file: /etc/ssl/prometheus/ca-crt.pem - cert_file: /etc/ssl/prometheus/server/scrape-crt.pem - key_file: /etc/ssl/prometheus/server/scrape-key.pem - file_sd_configs: - - files: - - "/etc/prometheus/targets/{{ job }}/*.yml" diff --git a/roles/monitoring/prometheus/server/templates/targets/blackbox/https.yml.j2 b/roles/monitoring/prometheus/server/templates/targets/blackbox/https.yml.j2 deleted file mode 100644 index 29c89590..00000000 --- a/roles/monitoring/prometheus/server/templates/targets/blackbox/https.yml.j2 +++ /dev/null @@ -1,4 +0,0 @@ -- targets: [ '{{ hostvars[target.config.exporter_hostname].prometheus_scrape_endpoint }}' ] - labels: - instance: '{{ target.instance }}' - __param_target: '{{ target.config.address }}' diff --git a/roles/monitoring/prometheus/server/templates/targets/blackbox/ping.yml.j2 b/roles/monitoring/prometheus/server/templates/targets/blackbox/ping.yml.j2 deleted file mode 100644 index 29c89590..00000000 --- a/roles/monitoring/prometheus/server/templates/targets/blackbox/ping.yml.j2 +++ /dev/null @@ -1,4 +0,0 @@ -- targets: [ '{{ hostvars[target.config.exporter_hostname].prometheus_scrape_endpoint }}' ] - labels: - instance: '{{ target.instance }}' - __param_target: '{{ target.config.address }}' diff --git a/roles/monitoring/prometheus/server/templates/targets/blackbox/probe.yml.j2 b/roles/monitoring/prometheus/server/templates/targets/blackbox/probe.yml.j2 new file mode 100644 index 00000000..4e336873 --- /dev/null +++ b/roles/monitoring/prometheus/server/templates/targets/blackbox/probe.yml.j2 @@ -0,0 +1,5 @@ +- targets: [ '{{ hostvars[target.exporter_hostname].prometheus_scrape_endpoint }}' ] + labels: + instance: '{{ target.instance }}' + __param_target: '{{ target.config.target }}' + __param_module: '{{ target.config.module }}' diff --git a/roles/monitoring/prometheus/server/templates/targets/blackbox/ssh.yml.j2 b/roles/monitoring/prometheus/server/templates/targets/blackbox/ssh.yml.j2 deleted file mode 100644 index 29c89590..00000000 --- a/roles/monitoring/prometheus/server/templates/targets/blackbox/ssh.yml.j2 +++ /dev/null @@ -1,4 +0,0 @@ -- targets: [ '{{ hostvars[target.config.exporter_hostname].prometheus_scrape_endpoint }}' ] - labels: - instance: '{{ target.instance }}' - __param_target: '{{ target.config.address }}' diff --git a/roles/monitoring/prometheus/server/templates/targets/nut/ups.yml.j2 b/roles/monitoring/prometheus/server/templates/targets/nut/ups.yml.j2 index 6003cd46..c60077c7 100644 --- a/roles/monitoring/prometheus/server/templates/targets/nut/ups.yml.j2 +++ b/roles/monitoring/prometheus/server/templates/targets/nut/ups.yml.j2 @@ -1,4 +1,4 @@ -- targets: [ '{{ hostvars[target.config.exporter_hostname].prometheus_scrape_endpoint }}' ] +- targets: [ '{{ hostvars[target.exporter_hostname].prometheus_scrape_endpoint }}' ] labels: instance: '{{ target.instance }}' __param_ups: '{{ target.config.ups }}' -- cgit v1.2.3 From 419ede2858769e4414a23a42b57931b83cf70d8c Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Sun, 26 Sep 2021 04:08:58 +0200 Subject: add job configs for ipmi and ipmi/remote --- inventory/host_vars/ch-testvm-prometheus.yml | 3 --- roles/monitoring/prometheus/exporter/ipmi/defaults/main.yml | 3 ++- roles/monitoring/prometheus/server/defaults/main/main.yml | 2 ++ roles/monitoring/prometheus/server/defaults/main/rules_ipmi.yml | 4 ++++ .../prometheus/server/defaults/main/rules_ipmi__remote.yml | 4 ++++ .../prometheus/server/templates/targets/ipmi/remote.yml.j2 | 5 +++++ 6 files changed, 17 insertions(+), 4 deletions(-) create mode 100644 roles/monitoring/prometheus/server/defaults/main/rules_ipmi.yml create mode 100644 roles/monitoring/prometheus/server/defaults/main/rules_ipmi__remote.yml create mode 100644 roles/monitoring/prometheus/server/templates/targets/ipmi/remote.yml.j2 (limited to 'inventory') diff --git a/inventory/host_vars/ch-testvm-prometheus.yml b/inventory/host_vars/ch-testvm-prometheus.yml index 939fa398..e539735f 100644 --- a/inventory/host_vars/ch-testvm-prometheus.yml +++ b/inventory/host_vars/ch-testvm-prometheus.yml @@ -36,9 +36,6 @@ network: spreadspace_apt_repo_components: - prometheus -prometheus_exporters_extra: - - ipmi - containerd_storage: type: lvm diff --git a/roles/monitoring/prometheus/exporter/ipmi/defaults/main.yml b/roles/monitoring/prometheus/exporter/ipmi/defaults/main.yml index 9b99f9a5..6cf14f76 100644 --- a/roles/monitoring/prometheus/exporter/ipmi/defaults/main.yml +++ b/roles/monitoring/prometheus/exporter/ipmi/defaults/main.yml @@ -1,6 +1,7 @@ --- prometheus_exporter_ipmi_modules: - default: {} + default: + collectors: [] # collectors: # - bmc # - ipmi diff --git a/roles/monitoring/prometheus/server/defaults/main/main.yml b/roles/monitoring/prometheus/server/defaults/main/main.yml index 7781fd69..1e0ccf78 100644 --- a/roles/monitoring/prometheus/server/defaults/main/main.yml +++ b/roles/monitoring/prometheus/server/defaults/main/main.yml @@ -18,6 +18,8 @@ prometheus_server_rules: nut/ups: "{{ prometheus_server_rules_nut__ups + prometheus_server_rules_nut__ups_extra }}" blackbox: "{{ prometheus_server_rules_blackbox + prometheus_server_rules_blackbox_extra }}" blackbox/probe: "{{ prometheus_server_rules_blackbox__probe + prometheus_server_rules_blackbox__probe_extra }}" + ipmi: "{{ prometheus_server_rules_ipmi + prometheus_server_rules_ipmi_extra }}" + ipmi/remote: "{{ prometheus_server_rules_ipmi__remote + prometheus_server_rules_ipmi__remote_extra }}" # prometheus_server_alertmanager: # url: "127.0.0.1:9093" diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_ipmi.yml b/roles/monitoring/prometheus/server/defaults/main/rules_ipmi.yml new file mode 100644 index 00000000..41dcd7e9 --- /dev/null +++ b/roles/monitoring/prometheus/server/defaults/main/rules_ipmi.yml @@ -0,0 +1,4 @@ +--- +prometheus_server_rules_ipmi_extra: [] +prometheus_server_rules_ipmi: [] +## TODO: add common IPMI alert rules diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_ipmi__remote.yml b/roles/monitoring/prometheus/server/defaults/main/rules_ipmi__remote.yml new file mode 100644 index 00000000..1f9338ea --- /dev/null +++ b/roles/monitoring/prometheus/server/defaults/main/rules_ipmi__remote.yml @@ -0,0 +1,4 @@ +--- +prometheus_server_rules_ipmi__remote_extra: [] +prometheus_server_rules_ipmi__remote: [] +## TODO: add remote-IPMI specific alert rules diff --git a/roles/monitoring/prometheus/server/templates/targets/ipmi/remote.yml.j2 b/roles/monitoring/prometheus/server/templates/targets/ipmi/remote.yml.j2 new file mode 100644 index 00000000..4e336873 --- /dev/null +++ b/roles/monitoring/prometheus/server/templates/targets/ipmi/remote.yml.j2 @@ -0,0 +1,5 @@ +- targets: [ '{{ hostvars[target.exporter_hostname].prometheus_scrape_endpoint }}' ] + labels: + instance: '{{ target.instance }}' + __param_target: '{{ target.config.target }}' + __param_module: '{{ target.config.module }}' -- cgit v1.2.3