From ad08b01391c404d4e0356467fc627d711ece8916 Mon Sep 17 00:00:00 2001
From: Christian Pointner <equinox@spreadspace.org>
Date: Wed, 22 Sep 2021 01:25:59 +0200
Subject: prometheus: no more special jobs

---
 inventory/host_vars/ch-mon.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'inventory/host_vars')

diff --git a/inventory/host_vars/ch-mon.yml b/inventory/host_vars/ch-mon.yml
index 7d8e334b..a211d4bb 100644
--- a/inventory/host_vars/ch-mon.yml
+++ b/inventory/host_vars/ch-mon.yml
@@ -76,7 +76,7 @@ prometheus_exporter_blackbox_modules_extra:
   icmp:
     prober: icmp
 
-prometheus_special_job_blackbox_ping:
+prometheus_job_blackbox_ping:
   - exporter_hostname: ch-mon
     instance: "ping-magentagw"
     address: 62.99.185.129
@@ -84,12 +84,12 @@ prometheus_special_job_blackbox_ping:
     instance: "ping-quad9"
     address: 9.9.9.9
 
-prometheus_special_job_blackbox_https:
+prometheus_job_blackbox_https:
   - exporter_hostname: ch-mon
     instance: "https-web.chaos-at-home.org"
     address: web.chaos-at-home.org
 
-prometheus_special_job_blackbox_ssh:
+prometheus_job_blackbox_ssh:
   - exporter_hostname: ch-mon
     instance: "ssh-{{ inventory_hostname }}"
     address: "{{ network_zones.svc.prefix | ipaddr(network_zones.svc.offsets[inventory_hostname]) | ipaddr('address') }}:{{ ansible_port | default(22) }}"
-- 
cgit v1.2.3


From ef4432c51bacb5b92c03a42cb1ea7f9d837ec8b6 Mon Sep 17 00:00:00 2001
From: Christian Pointner <equinox@spreadspace.org>
Date: Thu, 23 Sep 2021 16:50:24 +0200
Subject: use / as spereator for jobs formerly known as special

---
 inventory/group_vars/chaos-at-home-ups/vars.yml         |  2 +-
 inventory/group_vars/ele-ups/vars.yml                   |  2 +-
 inventory/group_vars/promzone-chaos-at-home/vars.yml    |  8 ++++----
 inventory/group_vars/promzone-elevate-festival/vars.yml |  2 +-
 inventory/host_vars/ch-mon.yml                          |  6 +++---
 .../monitoring/prometheus/server/defaults/main/main.yml |  8 ++++----
 .../server/defaults/main/rules_blackbox-https.yml       |  3 ---
 .../server/defaults/main/rules_blackbox-ping.yml        |  3 ---
 .../server/defaults/main/rules_blackbox-ssh.yml         |  3 ---
 .../server/defaults/main/rules_blackbox__https.yml      |  3 +++
 .../server/defaults/main/rules_blackbox__ping.yml       |  3 +++
 .../server/defaults/main/rules_blackbox__ssh.yml        |  3 +++
 .../prometheus/server/defaults/main/rules_nut-ups.yml   |  3 ---
 .../prometheus/server/defaults/main/rules_nut__ups.yml  |  3 +++
 .../prometheus/server/filter_plugins/prometheus.py      |  2 +-
 roles/monitoring/prometheus/server/tasks/main.yml       |  8 +++++++-
 .../server/templates/job-snippets/blackbox-https.j2     | 14 --------------
 .../server/templates/job-snippets/blackbox-ping.j2      | 14 --------------
 .../server/templates/job-snippets/blackbox-ssh.j2       | 14 --------------
 .../server/templates/job-snippets/blackbox/https.j2     | 14 ++++++++++++++
 .../server/templates/job-snippets/blackbox/ping.j2      | 14 ++++++++++++++
 .../server/templates/job-snippets/blackbox/ssh.j2       | 14 ++++++++++++++
 .../prometheus/server/templates/job-snippets/nut-ups.j2 | 13 -------------
 .../prometheus/server/templates/job-snippets/nut/ups.j2 | 13 +++++++++++++
 .../prometheus/server/templates/prometheus.yml.j2       |  5 ++++-
 .../server/templates/targets/blackbox-https.yml.j2      |  4 ----
 .../server/templates/targets/blackbox-ping.yml.j2       |  4 ----
 .../server/templates/targets/blackbox-ssh.yml.j2        |  4 ----
 .../server/templates/targets/blackbox/https.yml.j2      |  4 ++++
 .../server/templates/targets/blackbox/ping.yml.j2       |  4 ++++
 .../server/templates/targets/blackbox/ssh.yml.j2        |  4 ++++
 .../prometheus/server/templates/targets/nut-ups.yml.j2  | 17 -----------------
 .../prometheus/server/templates/targets/nut/ups.yml.j2  | 17 +++++++++++++++++
 33 files changed, 122 insertions(+), 113 deletions(-)
 delete mode 100644 roles/monitoring/prometheus/server/defaults/main/rules_blackbox-https.yml
 delete mode 100644 roles/monitoring/prometheus/server/defaults/main/rules_blackbox-ping.yml
 delete mode 100644 roles/monitoring/prometheus/server/defaults/main/rules_blackbox-ssh.yml
 create mode 100644 roles/monitoring/prometheus/server/defaults/main/rules_blackbox__https.yml
 create mode 100644 roles/monitoring/prometheus/server/defaults/main/rules_blackbox__ping.yml
 create mode 100644 roles/monitoring/prometheus/server/defaults/main/rules_blackbox__ssh.yml
 delete mode 100644 roles/monitoring/prometheus/server/defaults/main/rules_nut-ups.yml
 create mode 100644 roles/monitoring/prometheus/server/defaults/main/rules_nut__ups.yml
 delete mode 100644 roles/monitoring/prometheus/server/templates/job-snippets/blackbox-https.j2
 delete mode 100644 roles/monitoring/prometheus/server/templates/job-snippets/blackbox-ping.j2
 delete mode 100644 roles/monitoring/prometheus/server/templates/job-snippets/blackbox-ssh.j2
 create mode 100644 roles/monitoring/prometheus/server/templates/job-snippets/blackbox/https.j2
 create mode 100644 roles/monitoring/prometheus/server/templates/job-snippets/blackbox/ping.j2
 create mode 100644 roles/monitoring/prometheus/server/templates/job-snippets/blackbox/ssh.j2
 delete mode 100644 roles/monitoring/prometheus/server/templates/job-snippets/nut-ups.j2
 create mode 100644 roles/monitoring/prometheus/server/templates/job-snippets/nut/ups.j2
 delete mode 100644 roles/monitoring/prometheus/server/templates/targets/blackbox-https.yml.j2
 delete mode 100644 roles/monitoring/prometheus/server/templates/targets/blackbox-ping.yml.j2
 delete mode 100644 roles/monitoring/prometheus/server/templates/targets/blackbox-ssh.yml.j2
 create mode 100644 roles/monitoring/prometheus/server/templates/targets/blackbox/https.yml.j2
 create mode 100644 roles/monitoring/prometheus/server/templates/targets/blackbox/ping.yml.j2
 create mode 100644 roles/monitoring/prometheus/server/templates/targets/blackbox/ssh.yml.j2
 delete mode 100644 roles/monitoring/prometheus/server/templates/targets/nut-ups.yml.j2
 create mode 100644 roles/monitoring/prometheus/server/templates/targets/nut/ups.yml.j2

(limited to 'inventory/host_vars')

diff --git a/inventory/group_vars/chaos-at-home-ups/vars.yml b/inventory/group_vars/chaos-at-home-ups/vars.yml
index 99868165..f8c1bdf1 100644
--- a/inventory/group_vars/chaos-at-home-ups/vars.yml
+++ b/inventory/group_vars/chaos-at-home-ups/vars.yml
@@ -11,7 +11,7 @@ prometheus_scrape_endpoint: "{{ network_mgmt_zone.prefix | ipaddr(network_mgmt_z
 prometheus_exporters_default:
   - openwrt
 
-prometheus_job_nut_ups:
+prometheus_job_nut__ups:
   - exporter_hostname: ch-mon
     instance: "ups-{{ ups_name }}"
     ups: "{{ ups_name }}"
diff --git a/inventory/group_vars/ele-ups/vars.yml b/inventory/group_vars/ele-ups/vars.yml
index 0d22f770..1c4613a3 100644
--- a/inventory/group_vars/ele-ups/vars.yml
+++ b/inventory/group_vars/ele-ups/vars.yml
@@ -14,7 +14,7 @@ prometheus_scrape_endpoint: "{{ network_mgmt_zone.prefix | ipaddr(network_mgmt_z
 prometheus_exporters_default:
   - openwrt
 
-prometheus_job_nut_ups:
+prometheus_job_nut__ups:
   exporter_hostname: ele-mon
   instance: "ups-{{ ups_name }}"
   ups: "{{ ups_name }}"
diff --git a/inventory/group_vars/promzone-chaos-at-home/vars.yml b/inventory/group_vars/promzone-chaos-at-home/vars.yml
index d1958d47..84ed1263 100644
--- a/inventory/group_vars/promzone-chaos-at-home/vars.yml
+++ b/inventory/group_vars/promzone-chaos-at-home/vars.yml
@@ -9,10 +9,10 @@ prometheus_server: ch-mon
 prometheus_server_jobs:
   - node
   - openwrt
-  - nut-ups
-  - blackbox-ping
-  - blackbox-https
-  - blackbox-ssh
+  - nut/ups
+  - blackbox/ping
+  - blackbox/https
+  - blackbox/ssh
 
 prometheus_zone_name: chaos@home
 prometheus_zone_targets: "{{ groups['promzone-chaos-at-home'] }}"
diff --git a/inventory/group_vars/promzone-elevate-festival/vars.yml b/inventory/group_vars/promzone-elevate-festival/vars.yml
index a65a0cb7..43115dc4 100644
--- a/inventory/group_vars/promzone-elevate-festival/vars.yml
+++ b/inventory/group_vars/promzone-elevate-festival/vars.yml
@@ -9,7 +9,7 @@ prometheus_server: ele-mon
 prometheus_server_jobs:
   - node
   - openwrt
-  - nut-ups
+  - nut/ups
 
 prometheus_zone_name: Elevate Festival
 prometheus_zone_targets: "{{ groups['promzone-elevate-festival'] }}"
diff --git a/inventory/host_vars/ch-mon.yml b/inventory/host_vars/ch-mon.yml
index a211d4bb..242c4835 100644
--- a/inventory/host_vars/ch-mon.yml
+++ b/inventory/host_vars/ch-mon.yml
@@ -76,7 +76,7 @@ prometheus_exporter_blackbox_modules_extra:
   icmp:
     prober: icmp
 
-prometheus_job_blackbox_ping:
+prometheus_job_blackbox__ping:
   - exporter_hostname: ch-mon
     instance: "ping-magentagw"
     address: 62.99.185.129
@@ -84,12 +84,12 @@ prometheus_job_blackbox_ping:
     instance: "ping-quad9"
     address: 9.9.9.9
 
-prometheus_job_blackbox_https:
+prometheus_job_blackbox__https:
   - exporter_hostname: ch-mon
     instance: "https-web.chaos-at-home.org"
     address: web.chaos-at-home.org
 
-prometheus_job_blackbox_ssh:
+prometheus_job_blackbox__ssh:
   - exporter_hostname: ch-mon
     instance: "ssh-{{ inventory_hostname }}"
     address: "{{ network_zones.svc.prefix | ipaddr(network_zones.svc.offsets[inventory_hostname]) | ipaddr('address') }}:{{ ansible_port | default(22) }}"
diff --git a/roles/monitoring/prometheus/server/defaults/main/main.yml b/roles/monitoring/prometheus/server/defaults/main/main.yml
index 5be3ecd3..bae0cdba 100644
--- a/roles/monitoring/prometheus/server/defaults/main/main.yml
+++ b/roles/monitoring/prometheus/server/defaults/main/main.yml
@@ -14,10 +14,10 @@ prometheus_server_rules:
   prometheus: "{{ prometheus_server_rules_prometheus + ((prometheus_server_alertmanager is defined) | ternary(prometheus_server_rules_prometheus_alertmanager, [])) + prometheus_server_rules_prometheus_extra }}"
   node: "{{ prometheus_server_rules_node + prometheus_server_rules_node_extra }}"
   openwrt: "{{ prometheus_server_rules_openwrt + prometheus_server_rules_node_extra }}"
-  "nut-ups": "{{ prometheus_server_rules_nut_ups + prometheus_server_rules_nut_ups_extra }}"
-  "blackbox-ping": "{{ prometheus_server_rules_blackbox_ping + prometheus_server_rules_blackbox_ping_extra }}"
-  "blackbox-https": "{{ prometheus_server_rules_blackbox_https + prometheus_server_rules_blackbox_https_extra }}"
-  "blackbox-ssh": "{{ prometheus_server_rules_blackbox_ssh + prometheus_server_rules_blackbox_ssh_extra }}"
+  nut/ups: "{{ prometheus_server_rules_nut__ups + prometheus_server_rules_nut__ups_extra }}"
+  blackbox/ping: "{{ prometheus_server_rules_blackbox__ping + prometheus_server_rules_blackbox__ping_extra }}"
+  blackbox/https: "{{ prometheus_server_rules_blackbox__https + prometheus_server_rules_blackbox__https_extra }}"
+  blackbox/ssh: "{{ prometheus_server_rules_blackbox__ssh + prometheus_server_rules_blackbox__ssh_extra }}"
 
 # prometheus_server_alertmanager:
 #   url: "127.0.0.1:9093"
diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_blackbox-https.yml b/roles/monitoring/prometheus/server/defaults/main/rules_blackbox-https.yml
deleted file mode 100644
index bb806075..00000000
--- a/roles/monitoring/prometheus/server/defaults/main/rules_blackbox-https.yml
+++ /dev/null
@@ -1,3 +0,0 @@
----
-prometheus_server_rules_blackbox_https_extra: []
-prometheus_server_rules_blackbox_https: []
diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_blackbox-ping.yml b/roles/monitoring/prometheus/server/defaults/main/rules_blackbox-ping.yml
deleted file mode 100644
index 56c122f5..00000000
--- a/roles/monitoring/prometheus/server/defaults/main/rules_blackbox-ping.yml
+++ /dev/null
@@ -1,3 +0,0 @@
----
-prometheus_server_rules_blackbox_ping_extra: []
-prometheus_server_rules_blackbox_ping: []
diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_blackbox-ssh.yml b/roles/monitoring/prometheus/server/defaults/main/rules_blackbox-ssh.yml
deleted file mode 100644
index 727d2292..00000000
--- a/roles/monitoring/prometheus/server/defaults/main/rules_blackbox-ssh.yml
+++ /dev/null
@@ -1,3 +0,0 @@
----
-prometheus_server_rules_blackbox_ssh_extra: []
-prometheus_server_rules_blackbox_ssh: []
diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_blackbox__https.yml b/roles/monitoring/prometheus/server/defaults/main/rules_blackbox__https.yml
new file mode 100644
index 00000000..cfdc10bd
--- /dev/null
+++ b/roles/monitoring/prometheus/server/defaults/main/rules_blackbox__https.yml
@@ -0,0 +1,3 @@
+---
+prometheus_server_rules_blackbox__https_extra: []
+prometheus_server_rules_blackbox__https: []
diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_blackbox__ping.yml b/roles/monitoring/prometheus/server/defaults/main/rules_blackbox__ping.yml
new file mode 100644
index 00000000..06ce8607
--- /dev/null
+++ b/roles/monitoring/prometheus/server/defaults/main/rules_blackbox__ping.yml
@@ -0,0 +1,3 @@
+---
+prometheus_server_rules_blackbox__ping_extra: []
+prometheus_server_rules_blackbox__ping: []
diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_blackbox__ssh.yml b/roles/monitoring/prometheus/server/defaults/main/rules_blackbox__ssh.yml
new file mode 100644
index 00000000..8e717c41
--- /dev/null
+++ b/roles/monitoring/prometheus/server/defaults/main/rules_blackbox__ssh.yml
@@ -0,0 +1,3 @@
+---
+prometheus_server_rules_blackbox__ssh_extra: []
+prometheus_server_rules_blackbox__ssh: []
diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_nut-ups.yml b/roles/monitoring/prometheus/server/defaults/main/rules_nut-ups.yml
deleted file mode 100644
index 842007b4..00000000
--- a/roles/monitoring/prometheus/server/defaults/main/rules_nut-ups.yml
+++ /dev/null
@@ -1,3 +0,0 @@
----
-prometheus_server_rules_nut_ups_extra: []
-prometheus_server_rules_nut_ups: []
diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_nut__ups.yml b/roles/monitoring/prometheus/server/defaults/main/rules_nut__ups.yml
new file mode 100644
index 00000000..bccb0ca8
--- /dev/null
+++ b/roles/monitoring/prometheus/server/defaults/main/rules_nut__ups.yml
@@ -0,0 +1,3 @@
+---
+prometheus_server_rules_nut__ups_extra: []
+prometheus_server_rules_nut__ups: []
diff --git a/roles/monitoring/prometheus/server/filter_plugins/prometheus.py b/roles/monitoring/prometheus/server/filter_plugins/prometheus.py
index 056d216f..1443e837 100644
--- a/roles/monitoring/prometheus/server/filter_plugins/prometheus.py
+++ b/roles/monitoring/prometheus/server/filter_plugins/prometheus.py
@@ -11,7 +11,7 @@ def prometheus_job_targets(hostvars, jobs, targets):
         result = []
         for job in jobs:
             for target in targets:
-                special_config_varname = 'prometheus_job_' + job.replace('-', '_')
+                special_config_varname = 'prometheus_job_' + job.replace('-', '_').replace('/', '__')
                 if special_config_varname in hostvars[target]:
                     for config in hostvars[target][special_config_varname]:
                         result.append({'job': job, 'instance': config['instance'], 'config': config, 'enabled': True})
diff --git a/roles/monitoring/prometheus/server/tasks/main.yml b/roles/monitoring/prometheus/server/tasks/main.yml
index 4bcaa2d5..16167c9c 100644
--- a/roles/monitoring/prometheus/server/tasks/main.yml
+++ b/roles/monitoring/prometheus/server/tasks/main.yml
@@ -44,7 +44,7 @@
     path: "/etc/prometheus/{{ item }}"
     state: directory
 
-- name: create sub-directroy for all jobs in targets directory
+- name: create sub-directories for all jobs in targets directory
   loop: "{{ prometheus_server_jobs }}"
   file:
     path: "/etc/prometheus/targets/{{ item }}"
@@ -76,6 +76,12 @@
       state: absent
     notify: reload prometheus
 
+- name: create sub-directories for all jobs in rules directory
+  loop: "{{ prometheus_server_jobs | select('match', '.*/.*') | map('dirname') | unique }}"
+  file:
+    path: "/etc/prometheus/rules/{{ item }}"
+    state: directory
+
 - name: generate rules files for all jobs
   loop: "{{ prometheus_server_jobs | union(['prometheus']) }}"
   template:
diff --git a/roles/monitoring/prometheus/server/templates/job-snippets/blackbox-https.j2 b/roles/monitoring/prometheus/server/templates/job-snippets/blackbox-https.j2
deleted file mode 100644
index 98a64121..00000000
--- a/roles/monitoring/prometheus/server/templates/job-snippets/blackbox-https.j2
+++ /dev/null
@@ -1,14 +0,0 @@
-  - job_name: '{{ job }}'
-    metrics_path: /proxy
-    params:
-      module:
-      - blackbox
-      - http_tls_2xx
-    scheme: https
-    tls_config:
-      ca_file: /etc/ssl/prometheus/ca-crt.pem
-      cert_file: /etc/ssl/prometheus/server/scrape-crt.pem
-      key_file: /etc/ssl/prometheus/server/scrape-key.pem
-    file_sd_configs:
-    - files:
-      - "/etc/prometheus/targets/{{ job }}/*.yml"
diff --git a/roles/monitoring/prometheus/server/templates/job-snippets/blackbox-ping.j2 b/roles/monitoring/prometheus/server/templates/job-snippets/blackbox-ping.j2
deleted file mode 100644
index 736ffec1..00000000
--- a/roles/monitoring/prometheus/server/templates/job-snippets/blackbox-ping.j2
+++ /dev/null
@@ -1,14 +0,0 @@
-  - job_name: '{{ job }}'
-    metrics_path: /proxy
-    params:
-      module:
-      - blackbox
-      - icmp
-    scheme: https
-    tls_config:
-      ca_file: /etc/ssl/prometheus/ca-crt.pem
-      cert_file: /etc/ssl/prometheus/server/scrape-crt.pem
-      key_file: /etc/ssl/prometheus/server/scrape-key.pem
-    file_sd_configs:
-    - files:
-      - "/etc/prometheus/targets/{{ job }}/*.yml"
diff --git a/roles/monitoring/prometheus/server/templates/job-snippets/blackbox-ssh.j2 b/roles/monitoring/prometheus/server/templates/job-snippets/blackbox-ssh.j2
deleted file mode 100644
index 166f37ad..00000000
--- a/roles/monitoring/prometheus/server/templates/job-snippets/blackbox-ssh.j2
+++ /dev/null
@@ -1,14 +0,0 @@
-  - job_name: '{{ job }}'
-    metrics_path: /proxy
-    params:
-      module:
-      - blackbox
-      - ssh_banner
-    scheme: https
-    tls_config:
-      ca_file: /etc/ssl/prometheus/ca-crt.pem
-      cert_file: /etc/ssl/prometheus/server/scrape-crt.pem
-      key_file: /etc/ssl/prometheus/server/scrape-key.pem
-    file_sd_configs:
-    - files:
-      - "/etc/prometheus/targets/{{ job }}/*.yml"
diff --git a/roles/monitoring/prometheus/server/templates/job-snippets/blackbox/https.j2 b/roles/monitoring/prometheus/server/templates/job-snippets/blackbox/https.j2
new file mode 100644
index 00000000..98a64121
--- /dev/null
+++ b/roles/monitoring/prometheus/server/templates/job-snippets/blackbox/https.j2
@@ -0,0 +1,14 @@
+  - job_name: '{{ job }}'
+    metrics_path: /proxy
+    params:
+      module:
+      - blackbox
+      - http_tls_2xx
+    scheme: https
+    tls_config:
+      ca_file: /etc/ssl/prometheus/ca-crt.pem
+      cert_file: /etc/ssl/prometheus/server/scrape-crt.pem
+      key_file: /etc/ssl/prometheus/server/scrape-key.pem
+    file_sd_configs:
+    - files:
+      - "/etc/prometheus/targets/{{ job }}/*.yml"
diff --git a/roles/monitoring/prometheus/server/templates/job-snippets/blackbox/ping.j2 b/roles/monitoring/prometheus/server/templates/job-snippets/blackbox/ping.j2
new file mode 100644
index 00000000..736ffec1
--- /dev/null
+++ b/roles/monitoring/prometheus/server/templates/job-snippets/blackbox/ping.j2
@@ -0,0 +1,14 @@
+  - job_name: '{{ job }}'
+    metrics_path: /proxy
+    params:
+      module:
+      - blackbox
+      - icmp
+    scheme: https
+    tls_config:
+      ca_file: /etc/ssl/prometheus/ca-crt.pem
+      cert_file: /etc/ssl/prometheus/server/scrape-crt.pem
+      key_file: /etc/ssl/prometheus/server/scrape-key.pem
+    file_sd_configs:
+    - files:
+      - "/etc/prometheus/targets/{{ job }}/*.yml"
diff --git a/roles/monitoring/prometheus/server/templates/job-snippets/blackbox/ssh.j2 b/roles/monitoring/prometheus/server/templates/job-snippets/blackbox/ssh.j2
new file mode 100644
index 00000000..166f37ad
--- /dev/null
+++ b/roles/monitoring/prometheus/server/templates/job-snippets/blackbox/ssh.j2
@@ -0,0 +1,14 @@
+  - job_name: '{{ job }}'
+    metrics_path: /proxy
+    params:
+      module:
+      - blackbox
+      - ssh_banner
+    scheme: https
+    tls_config:
+      ca_file: /etc/ssl/prometheus/ca-crt.pem
+      cert_file: /etc/ssl/prometheus/server/scrape-crt.pem
+      key_file: /etc/ssl/prometheus/server/scrape-key.pem
+    file_sd_configs:
+    - files:
+      - "/etc/prometheus/targets/{{ job }}/*.yml"
diff --git a/roles/monitoring/prometheus/server/templates/job-snippets/nut-ups.j2 b/roles/monitoring/prometheus/server/templates/job-snippets/nut-ups.j2
deleted file mode 100644
index 3a2c5c62..00000000
--- a/roles/monitoring/prometheus/server/templates/job-snippets/nut-ups.j2
+++ /dev/null
@@ -1,13 +0,0 @@
-  - job_name: '{{ job }}'
-    metrics_path: /proxy
-    params:
-      module:
-      - nut
-    scheme: https
-    tls_config:
-      ca_file: /etc/ssl/prometheus/ca-crt.pem
-      cert_file: /etc/ssl/prometheus/server/scrape-crt.pem
-      key_file: /etc/ssl/prometheus/server/scrape-key.pem
-    file_sd_configs:
-    - files:
-      - "/etc/prometheus/targets/{{ job }}/*.yml"
diff --git a/roles/monitoring/prometheus/server/templates/job-snippets/nut/ups.j2 b/roles/monitoring/prometheus/server/templates/job-snippets/nut/ups.j2
new file mode 100644
index 00000000..3a2c5c62
--- /dev/null
+++ b/roles/monitoring/prometheus/server/templates/job-snippets/nut/ups.j2
@@ -0,0 +1,13 @@
+  - job_name: '{{ job }}'
+    metrics_path: /proxy
+    params:
+      module:
+      - nut
+    scheme: https
+    tls_config:
+      ca_file: /etc/ssl/prometheus/ca-crt.pem
+      cert_file: /etc/ssl/prometheus/server/scrape-crt.pem
+      key_file: /etc/ssl/prometheus/server/scrape-key.pem
+    file_sd_configs:
+    - files:
+      - "/etc/prometheus/targets/{{ job }}/*.yml"
diff --git a/roles/monitoring/prometheus/server/templates/prometheus.yml.j2 b/roles/monitoring/prometheus/server/templates/prometheus.yml.j2
index 4a079896..8156341d 100644
--- a/roles/monitoring/prometheus/server/templates/prometheus.yml.j2
+++ b/roles/monitoring/prometheus/server/templates/prometheus.yml.j2
@@ -6,6 +6,9 @@ global:
 
 rule_files:
   - /etc/prometheus/rules/*.yml
+{% for subdir in (prometheus_server_jobs | select('match', '.*/.*') | map('dirname') | unique) %}
+  - /etc/prometheus/rules/{{ subdir }}/*.yml
+{% endfor %}
 {% if prometheus_server_alertmanager is defined %}
 
 alerting:
@@ -37,7 +40,7 @@ scrape_configs:
 {% endif %}
 {% for job in (prometheus_server_jobs) %}
 
-{%   include 'job-snippets/' + (lookup('first_found', {'paths': ['templates/job-snippets'], 'files': [job + '.j2', 'generic.j2']}) | basename) %}{{ '' }}
+{%   include lookup('first_found', {'paths': ['templates/job-snippets'], 'files': [job + '.j2', 'generic.j2']}) | relpath(template_fullpath | dirname) %}{{ '' }}
 {% endfor %}
 {% if prometheus_server_jobs_extra is defined %}
 
diff --git a/roles/monitoring/prometheus/server/templates/targets/blackbox-https.yml.j2 b/roles/monitoring/prometheus/server/templates/targets/blackbox-https.yml.j2
deleted file mode 100644
index b1a33df3..00000000
--- a/roles/monitoring/prometheus/server/templates/targets/blackbox-https.yml.j2
+++ /dev/null
@@ -1,4 +0,0 @@
-- targets: [ '{{ hostvars[target.config.exporter_hostname].prometheus_scrape_endpoint }}' ]
-  labels:
-    instance: '{{ target.instance }}'
-    __param_target: {{ target.config.address }}
diff --git a/roles/monitoring/prometheus/server/templates/targets/blackbox-ping.yml.j2 b/roles/monitoring/prometheus/server/templates/targets/blackbox-ping.yml.j2
deleted file mode 100644
index b1a33df3..00000000
--- a/roles/monitoring/prometheus/server/templates/targets/blackbox-ping.yml.j2
+++ /dev/null
@@ -1,4 +0,0 @@
-- targets: [ '{{ hostvars[target.config.exporter_hostname].prometheus_scrape_endpoint }}' ]
-  labels:
-    instance: '{{ target.instance }}'
-    __param_target: {{ target.config.address }}
diff --git a/roles/monitoring/prometheus/server/templates/targets/blackbox-ssh.yml.j2 b/roles/monitoring/prometheus/server/templates/targets/blackbox-ssh.yml.j2
deleted file mode 100644
index b1a33df3..00000000
--- a/roles/monitoring/prometheus/server/templates/targets/blackbox-ssh.yml.j2
+++ /dev/null
@@ -1,4 +0,0 @@
-- targets: [ '{{ hostvars[target.config.exporter_hostname].prometheus_scrape_endpoint }}' ]
-  labels:
-    instance: '{{ target.instance }}'
-    __param_target: {{ target.config.address }}
diff --git a/roles/monitoring/prometheus/server/templates/targets/blackbox/https.yml.j2 b/roles/monitoring/prometheus/server/templates/targets/blackbox/https.yml.j2
new file mode 100644
index 00000000..29c89590
--- /dev/null
+++ b/roles/monitoring/prometheus/server/templates/targets/blackbox/https.yml.j2
@@ -0,0 +1,4 @@
+- targets: [ '{{ hostvars[target.config.exporter_hostname].prometheus_scrape_endpoint }}' ]
+  labels:
+    instance: '{{ target.instance }}'
+    __param_target: '{{ target.config.address }}'
diff --git a/roles/monitoring/prometheus/server/templates/targets/blackbox/ping.yml.j2 b/roles/monitoring/prometheus/server/templates/targets/blackbox/ping.yml.j2
new file mode 100644
index 00000000..29c89590
--- /dev/null
+++ b/roles/monitoring/prometheus/server/templates/targets/blackbox/ping.yml.j2
@@ -0,0 +1,4 @@
+- targets: [ '{{ hostvars[target.config.exporter_hostname].prometheus_scrape_endpoint }}' ]
+  labels:
+    instance: '{{ target.instance }}'
+    __param_target: '{{ target.config.address }}'
diff --git a/roles/monitoring/prometheus/server/templates/targets/blackbox/ssh.yml.j2 b/roles/monitoring/prometheus/server/templates/targets/blackbox/ssh.yml.j2
new file mode 100644
index 00000000..29c89590
--- /dev/null
+++ b/roles/monitoring/prometheus/server/templates/targets/blackbox/ssh.yml.j2
@@ -0,0 +1,4 @@
+- targets: [ '{{ hostvars[target.config.exporter_hostname].prometheus_scrape_endpoint }}' ]
+  labels:
+    instance: '{{ target.instance }}'
+    __param_target: '{{ target.config.address }}'
diff --git a/roles/monitoring/prometheus/server/templates/targets/nut-ups.yml.j2 b/roles/monitoring/prometheus/server/templates/targets/nut-ups.yml.j2
deleted file mode 100644
index d63d79a7..00000000
--- a/roles/monitoring/prometheus/server/templates/targets/nut-ups.yml.j2
+++ /dev/null
@@ -1,17 +0,0 @@
-- targets: [ '{{ hostvars[target.config.exporter_hostname].prometheus_scrape_endpoint }}' ]
-  labels:
-    instance: '{{ target.instance }}'
-    __param_ups: {{ target.config.ups }}
-    __param_server: {{ target.config.server | default('127.0.0.1') }}
-{% if 'username' in target.config %}
-    __param_username: {{ target.config.username }}
-{% endif  %}
-{% if 'password' in target.config %}
-    __param_password: {{ target.config.password }}
-{% endif  %}
-{% if 'variables' in target.config %}
-    __param_variables: {{ target.config.variables }}
-{% endif  %}
-{% if 'statuses' in target.config %}
-    __param_statuses: {{ target.config.statuses }}
-{% endif  %}
diff --git a/roles/monitoring/prometheus/server/templates/targets/nut/ups.yml.j2 b/roles/monitoring/prometheus/server/templates/targets/nut/ups.yml.j2
new file mode 100644
index 00000000..6003cd46
--- /dev/null
+++ b/roles/monitoring/prometheus/server/templates/targets/nut/ups.yml.j2
@@ -0,0 +1,17 @@
+- targets: [ '{{ hostvars[target.config.exporter_hostname].prometheus_scrape_endpoint }}' ]
+  labels:
+    instance: '{{ target.instance }}'
+    __param_ups: '{{ target.config.ups }}'
+    __param_server: '{{ target.config.server | default('127.0.0.1') }}'
+{% if 'username' in target.config %}
+    __param_username: '{{ target.config.username }}'
+{% endif  %}
+{% if 'password' in target.config %}
+    __param_password: '{{ target.config.password }}'
+{% endif  %}
+{% if 'variables' in target.config %}
+    __param_variables: '{{ target.config.variables }}'
+{% endif  %}
+{% if 'statuses' in target.config %}
+    __param_statuses: '{{ target.config.statuses }}'
+{% endif  %}
-- 
cgit v1.2.3


From ec55b1572702b91184d99fec89fec537cfe2ea1f Mon Sep 17 00:00:00 2001
From: Christian Pointner <equinox@spreadspace.org>
Date: Thu, 23 Sep 2021 20:18:08 +0200
Subject: force blackbox exporter to ipv4 by default

---
 inventory/host_vars/ch-mon.yml                                  | 7 +++++--
 roles/monitoring/prometheus/exporter/blackbox/defaults/main.yml | 7 +++++++
 2 files changed, 12 insertions(+), 2 deletions(-)

(limited to 'inventory/host_vars')

diff --git a/inventory/host_vars/ch-mon.yml b/inventory/host_vars/ch-mon.yml
index 242c4835..d1a710b9 100644
--- a/inventory/host_vars/ch-mon.yml
+++ b/inventory/host_vars/ch-mon.yml
@@ -86,8 +86,11 @@ prometheus_job_blackbox__ping:
 
 prometheus_job_blackbox__https:
   - exporter_hostname: ch-mon
-    instance: "https-web.chaos-at-home.org"
-    address: web.chaos-at-home.org
+    instance: "https-pan.chaos-at-home.org"
+    address: "https://pan.chaos-at-home.org"
+  - exporter_hostname: ch-mon
+    instance: "https-mimas.chaos-at-home.org"
+    address: "https://mimas.chaos-at-home.org"
 
 prometheus_job_blackbox__ssh:
   - exporter_hostname: ch-mon
diff --git a/roles/monitoring/prometheus/exporter/blackbox/defaults/main.yml b/roles/monitoring/prometheus/exporter/blackbox/defaults/main.yml
index 4e7d8d9a..73b9fde1 100644
--- a/roles/monitoring/prometheus/exporter/blackbox/defaults/main.yml
+++ b/roles/monitoring/prometheus/exporter/blackbox/defaults/main.yml
@@ -2,23 +2,30 @@
 prometheus_exporter_blackbox_modules:
   tcp_connect:
     prober: tcp
+    tcp:
+      preferred_ip_protocol: "ip4"
   tcp_tls_connect:
     prober: tcp
     tcp:
+      preferred_ip_protocol: "ip4"
       tls: true
       tls_config:
         insecure_skip_verify: true
   http_2xx:
     prober: http
+    http:
+      preferred_ip_protocol: "ip4"
   http_tls_2xx:
     prober: http
     http:
+      preferred_ip_protocol: "ip4"
       fail_if_not_ssl: true
       tls_config:
         insecure_skip_verify: true
   ssh_banner:
     prober: tcp
     tcp:
+      preferred_ip_protocol: "ip4"
       query_response:
       - expect: "^SSH-2.0-"
       - send: "SSH-2.0-blackbox-ssh-check"
-- 
cgit v1.2.3


From 063bdb70a8e8353908ca9742e05be8fac65a61bf Mon Sep 17 00:00:00 2001
From: Christian Pointner <equinox@spreadspace.org>
Date: Sat, 25 Sep 2021 23:36:40 +0200
Subject: move away from exporter-exporter in favor for nginx

---
 chaos-at-home/ch-testvm-prometheus.yml             |  1 +
 inventory/host_vars/ch-testvm-prometheus.yml       |  3 ++
 .../prometheus/exporter/base/defaults/main.yml     |  2 +-
 .../prometheus/exporter/base/handlers/main.yml     |  6 ++--
 .../prometheus/exporter/base/tasks/main.yml        | 30 +++++---------------
 .../exporter/base/templates/nginx-vhost.j2         | 19 +++++++++++++
 .../prometheus/exporter/base/templates/service.j2  | 32 ----------------------
 .../prometheus/exporter/blackbox/handlers/main.yml |  7 ++---
 .../prometheus/exporter/blackbox/tasks/main.yml    | 11 ++++----
 .../prometheus/exporter/ipmi/handlers/main.yml     |  7 ++---
 .../prometheus/exporter/ipmi/tasks/main.yml        | 21 ++++++--------
 roles/monitoring/prometheus/exporter/meta/main.yml | 10 ++++---
 .../prometheus/exporter/mikrotik/handlers/main.yml |  7 ++---
 .../prometheus/exporter/mikrotik/tasks/main.yml    | 10 +++----
 .../prometheus/exporter/node/handlers/main.yml     |  7 ++---
 .../prometheus/exporter/node/tasks/main.yml        | 10 +++----
 .../prometheus/exporter/nut/handlers/main.yml      |  7 ++---
 .../prometheus/exporter/nut/tasks/main.yml         | 14 ++++++----
 .../server/templates/jobs/blackbox/https.j2        |  3 +-
 .../server/templates/jobs/blackbox/ping.j2         |  3 +-
 .../server/templates/jobs/blackbox/ssh.j2          |  3 +-
 .../prometheus/server/templates/jobs/generic.j2    |  5 +---
 .../prometheus/server/templates/jobs/node.j2       |  5 +---
 .../prometheus/server/templates/jobs/nut/ups.j2    |  5 +---
 24 files changed, 92 insertions(+), 136 deletions(-)
 create mode 100644 roles/monitoring/prometheus/exporter/base/templates/nginx-vhost.j2
 delete mode 100644 roles/monitoring/prometheus/exporter/base/templates/service.j2

(limited to 'inventory/host_vars')

diff --git a/chaos-at-home/ch-testvm-prometheus.yml b/chaos-at-home/ch-testvm-prometheus.yml
index 3fd99d41..c0f33b8f 100644
--- a/chaos-at-home/ch-testvm-prometheus.yml
+++ b/chaos-at-home/ch-testvm-prometheus.yml
@@ -7,6 +7,7 @@
   - role: core/sshd/base
   - role: core/zsh
   - role: core/ntp
+  - role: nginx/base
   - role: apt-repo/spreadspace
   - role: monitoring/prometheus/exporter
   # - role: kubernetes/base
diff --git a/inventory/host_vars/ch-testvm-prometheus.yml b/inventory/host_vars/ch-testvm-prometheus.yml
index e539735f..939fa398 100644
--- a/inventory/host_vars/ch-testvm-prometheus.yml
+++ b/inventory/host_vars/ch-testvm-prometheus.yml
@@ -36,6 +36,9 @@ network:
 spreadspace_apt_repo_components:
   - prometheus
 
+prometheus_exporters_extra:
+  - ipmi
+
 
 containerd_storage:
   type: lvm
diff --git a/roles/monitoring/prometheus/exporter/base/defaults/main.yml b/roles/monitoring/prometheus/exporter/base/defaults/main.yml
index 963763a5..613943d8 100644
--- a/roles/monitoring/prometheus/exporter/base/defaults/main.yml
+++ b/roles/monitoring/prometheus/exporter/base/defaults/main.yml
@@ -1,2 +1,2 @@
 ---
-prometheus_exporter_listen: ":9999"
+prometheus_exporter_listen: "9999"
diff --git a/roles/monitoring/prometheus/exporter/base/handlers/main.yml b/roles/monitoring/prometheus/exporter/base/handlers/main.yml
index ebd760cf..d4e42ca0 100644
--- a/roles/monitoring/prometheus/exporter/base/handlers/main.yml
+++ b/roles/monitoring/prometheus/exporter/base/handlers/main.yml
@@ -1,5 +1,5 @@
 ---
-- name: restart prometheus-exporter-exporter
+- name: reload nginx
   service:
-    name: prometheus-exporter-exporter
-    state: restarted
+    name: nginx
+    state: reloaded
diff --git a/roles/monitoring/prometheus/exporter/base/tasks/main.yml b/roles/monitoring/prometheus/exporter/base/tasks/main.yml
index eeb2a23d..5f42867d 100644
--- a/roles/monitoring/prometheus/exporter/base/tasks/main.yml
+++ b/roles/monitoring/prometheus/exporter/base/tasks/main.yml
@@ -6,17 +6,6 @@
     - spreadspace_apt_repo_components is defined
     - "'prometheus' in spreadspace_apt_repo_components"
 
-  ## TODO: pin version
-- name: install apt packages
-  apt:
-    name: prom-exporter-exporter
-    state: present
-
-- name: create configuration directories
-  file:
-    path: /etc/prometheus/exporter/exporter
-    state: directory
-
 - name: add user for prometheus-exporter
   user:
     name: prometheus-exporter
@@ -27,15 +16,10 @@
 - name: create TLS certificate and key
   import_tasks: tls.yml
 
-- name: generate systemd service unit
-  template:
-    src: service.j2
-    dest: /etc/systemd/system/prometheus-exporter-exporter.service
-  notify: restart prometheus-exporter-exporter
-
-- name: make sure prometheus-exporter-exporter is enabled and started
-  systemd:
-    name: prometheus-exporter-exporter.service
-    daemon_reload: yes
-    state: started
-    enabled: yes
+- name: configure nginx vhost
+  import_role:
+    name: nginx/vhost
+  vars:
+    nginx_vhost:
+      name: prometheus-exporter
+      content: "{{ lookup('template', 'nginx-vhost.j2') }}"
diff --git a/roles/monitoring/prometheus/exporter/base/templates/nginx-vhost.j2 b/roles/monitoring/prometheus/exporter/base/templates/nginx-vhost.j2
new file mode 100644
index 00000000..70e65b29
--- /dev/null
+++ b/roles/monitoring/prometheus/exporter/base/templates/nginx-vhost.j2
@@ -0,0 +1,19 @@
+server {
+    listen {{ prometheus_exporter_listen }} ssl;
+    server_name _;
+
+    ssl_certificate /etc/ssl/prometheus/exporter/crt.pem;
+    ssl_certificate_key /etc/ssl/prometheus/exporter/key.pem;
+    ssl_client_certificate /etc/ssl/prometheus/ca-crt.pem;
+    ssl_verify_client on;
+
+    root /nonexistent;
+
+    location = / {
+        return 404 'please specify the exporter you want to reach!';
+    }
+
+    include snippets/proxy-nobuff.conf;
+
+    include /etc/prometheus/exporter/*.locations;
+}
diff --git a/roles/monitoring/prometheus/exporter/base/templates/service.j2 b/roles/monitoring/prometheus/exporter/base/templates/service.j2
deleted file mode 100644
index 3d44744a..00000000
--- a/roles/monitoring/prometheus/exporter/base/templates/service.j2
+++ /dev/null
@@ -1,32 +0,0 @@
-[Unit]
-Description=Prometheus exporter proxy
-
-[Service]
-Restart=always
-User=prometheus-exporter
-ExecStart=/usr/bin/prometheus-exporter-exporter -config.dirs=/etc/prometheus/exporter/exporter -config.file="" -web.listen-address="" -web.tls.listen-address="{{ prometheus_exporter_listen }}" -web.tls.cert="/etc/ssl/prometheus/exporter/crt.pem" -web.tls.key="/etc/ssl/prometheus/exporter/key.pem" --web.tls.ca="/etc/ssl/prometheus/ca-crt.pem" -web.tls.verify
-{# TODO: implement reloading once the exporter_exporter supports this #}
-
-# systemd hardening-options
-AmbientCapabilities=
-CapabilityBoundingSet=
-DeviceAllow=/dev/null rw
-DevicePolicy=strict
-LockPersonality=true
-MemoryDenyWriteExecute=true
-NoNewPrivileges=true
-PrivateDevices=true
-PrivateTmp=true
-PrivateUsers=true
-ProtectControlGroups=true
-ProtectHome=true
-ProtectKernelModules=true
-ProtectKernelTunables=true
-ProtectSystem=strict
-RemoveIPC=true
-RestrictNamespaces=true
-RestrictRealtime=true
-SystemCallArchitectures=native
-
-[Install]
-WantedBy=multi-user.target
diff --git a/roles/monitoring/prometheus/exporter/blackbox/handlers/main.yml b/roles/monitoring/prometheus/exporter/blackbox/handlers/main.yml
index 99a416e2..12250769 100644
--- a/roles/monitoring/prometheus/exporter/blackbox/handlers/main.yml
+++ b/roles/monitoring/prometheus/exporter/blackbox/handlers/main.yml
@@ -9,8 +9,7 @@
     name: prometheus-blackbox-exporter
     state: reloaded
 
-- name: reload prometheus-exporter-exporter
+- name: reload nginx
   service:
-    name: prometheus-exporter-exporter
-    ## TODO: implement reload once exporter_exporter supports this...
-    state: restarted
+    name: nginx
+    state: reloaded
diff --git a/roles/monitoring/prometheus/exporter/blackbox/tasks/main.yml b/roles/monitoring/prometheus/exporter/blackbox/tasks/main.yml
index 782c3561..f9793df6 100644
--- a/roles/monitoring/prometheus/exporter/blackbox/tasks/main.yml
+++ b/roles/monitoring/prometheus/exporter/blackbox/tasks/main.yml
@@ -32,9 +32,8 @@
 - name: register exporter
   copy:
     content: |
-      method: http
-      http:
-        port: 9115
-        path: /probe
-    dest: /etc/prometheus/exporter/exporter/blackbox.yml
-  notify: reload prometheus-exporter-exporter
+      location = /blackbox {
+          proxy_pass http://127.0.0.1:9115/probe;
+      }
+    dest: /etc/prometheus/exporter/blackbox.locations
+  notify: reload nginx
diff --git a/roles/monitoring/prometheus/exporter/ipmi/handlers/main.yml b/roles/monitoring/prometheus/exporter/ipmi/handlers/main.yml
index 40a945ae..a8eb55b3 100644
--- a/roles/monitoring/prometheus/exporter/ipmi/handlers/main.yml
+++ b/roles/monitoring/prometheus/exporter/ipmi/handlers/main.yml
@@ -9,8 +9,7 @@
     name: prometheus-ipmi-exporter
     state: reloaded
 
-- name: reload prometheus-exporter-exporter
+- name: reload nginx
   service:
-    name: prometheus-exporter-exporter
-    ## TODO: implement reload once exporter_exporter supports this...
-    state: restarted
+    name: nginx
+    state: reloaded
diff --git a/roles/monitoring/prometheus/exporter/ipmi/tasks/main.yml b/roles/monitoring/prometheus/exporter/ipmi/tasks/main.yml
index 9e63f692..91318f16 100644
--- a/roles/monitoring/prometheus/exporter/ipmi/tasks/main.yml
+++ b/roles/monitoring/prometheus/exporter/ipmi/tasks/main.yml
@@ -30,18 +30,13 @@
     enabled: yes
 
 - name: register exporter
-  loop:
-  - name: local
-    path: /metrics
-  - name: remote
-    path: /ipmi
-  loop_control:
-    label: "{{ item.name }}"
   copy:
     content: |
-      method: http
-      http:
-        port: 9290
-        path: {{ item.path }}
-    dest: "/etc/prometheus/exporter/exporter/ipmi-{{ item.name }}.yml"
-  notify: reload prometheus-exporter-exporter
+      location = /ipmi {
+          proxy_pass http://127.0.0.1:9290/metrics;
+      }
+      location = /ipmi/remote {
+          proxy_pass http://127.0.0.1:9290/ipmi;
+      }
+    dest: /etc/prometheus/exporter/ipmi.locations
+  notify: reload nginx
diff --git a/roles/monitoring/prometheus/exporter/meta/main.yml b/roles/monitoring/prometheus/exporter/meta/main.yml
index 22131422..68fce6cb 100644
--- a/roles/monitoring/prometheus/exporter/meta/main.yml
+++ b/roles/monitoring/prometheus/exporter/meta/main.yml
@@ -1,11 +1,13 @@
 ---
 dependencies:
   - role: monitoring/prometheus/exporter/base
-  - role: monitoring/prometheus/exporter/node
-    when: "'node' in (prometheus_exporters_default | union(prometheus_exporters_extra))"
   - role: monitoring/prometheus/exporter/blackbox
     when: "'blackbox' in (prometheus_exporters_default | union(prometheus_exporters_extra))"
-  - role: monitoring/prometheus/exporter/nut
-    when: "'nut' in (prometheus_exporters_default | union(prometheus_exporters_extra))"
+  - role: monitoring/prometheus/exporter/ipmi
+    when: "'ipmi' in (prometheus_exporters_default | union(prometheus_exporters_extra))"
   - role: monitoring/prometheus/exporter/mikrotik
     when: "'mikrotik' in (prometheus_exporters_default | union(prometheus_exporters_extra))"
+  - role: monitoring/prometheus/exporter/node
+    when: "'node' in (prometheus_exporters_default | union(prometheus_exporters_extra))"
+  - role: monitoring/prometheus/exporter/nut
+    when: "'nut' in (prometheus_exporters_default | union(prometheus_exporters_extra))"
diff --git a/roles/monitoring/prometheus/exporter/mikrotik/handlers/main.yml b/roles/monitoring/prometheus/exporter/mikrotik/handlers/main.yml
index cb85d0d9..c5844220 100644
--- a/roles/monitoring/prometheus/exporter/mikrotik/handlers/main.yml
+++ b/roles/monitoring/prometheus/exporter/mikrotik/handlers/main.yml
@@ -4,8 +4,7 @@
     name: prometheus-mikrotik-exporter
     state: restarted
 
-- name: reload prometheus-exporter-exporter
+- name: reload nginx
   service:
-    name: prometheus-exporter-exporter
-    ## TODO: implement reload once exporter_exporter supports this...
-    state: restarted
+    name: nginx
+    state: reloaded
diff --git a/roles/monitoring/prometheus/exporter/mikrotik/tasks/main.yml b/roles/monitoring/prometheus/exporter/mikrotik/tasks/main.yml
index 07219c68..72c78e4a 100644
--- a/roles/monitoring/prometheus/exporter/mikrotik/tasks/main.yml
+++ b/roles/monitoring/prometheus/exporter/mikrotik/tasks/main.yml
@@ -35,8 +35,8 @@
 - name: register exporter
   copy:
     content: |
-      method: http
-      http:
-        port: 9436
-    dest: /etc/prometheus/exporter/exporter/mikrotik.yml
-  notify: reload prometheus-exporter-exporter
+      location = /mikrotik {
+          proxy_pass http://127.0.0.1:9436/metrics;
+      }
+    dest: /etc/prometheus/exporter/mikrotik.locations
+  notify: reload nginx
diff --git a/roles/monitoring/prometheus/exporter/node/handlers/main.yml b/roles/monitoring/prometheus/exporter/node/handlers/main.yml
index 3e1b2000..56056ea6 100644
--- a/roles/monitoring/prometheus/exporter/node/handlers/main.yml
+++ b/roles/monitoring/prometheus/exporter/node/handlers/main.yml
@@ -4,8 +4,7 @@
     name: prometheus-node-exporter
     state: restarted
 
-- name: reload prometheus-exporter-exporter
+- name: reload nginx
   service:
-    name: prometheus-exporter-exporter
-    ## TODO: implement reload once exporter_exporter supports this...
-    state: restarted
+    name: nginx
+    state: reloaded
diff --git a/roles/monitoring/prometheus/exporter/node/tasks/main.yml b/roles/monitoring/prometheus/exporter/node/tasks/main.yml
index 56903a33..2811c759 100644
--- a/roles/monitoring/prometheus/exporter/node/tasks/main.yml
+++ b/roles/monitoring/prometheus/exporter/node/tasks/main.yml
@@ -28,11 +28,11 @@
 - name: register exporter
   copy:
     content: |
-      method: http
-      http:
-        port: 9100
-    dest: /etc/prometheus/exporter/exporter/node.yml
-  notify: reload prometheus-exporter-exporter
+      location = /node {
+          proxy_pass http://127.0.0.1:9100/metrics;
+      }
+    dest: /etc/prometheus/exporter/node.locations
+  notify: reload nginx
 
 - name: create directory for textfile collector scripts
   file:
diff --git a/roles/monitoring/prometheus/exporter/nut/handlers/main.yml b/roles/monitoring/prometheus/exporter/nut/handlers/main.yml
index 6e10f43b..edd87ed5 100644
--- a/roles/monitoring/prometheus/exporter/nut/handlers/main.yml
+++ b/roles/monitoring/prometheus/exporter/nut/handlers/main.yml
@@ -4,8 +4,7 @@
     name: prometheus-nut-exporter
     state: restarted
 
-- name: reload prometheus-exporter-exporter
+- name: reload ngnix
   service:
-    name: prometheus-exporter-exporter
-    ## TODO: implement reload once exporter_exporter supports this...
-    state: restarted
+    name: nginx
+    state: reloaded
diff --git a/roles/monitoring/prometheus/exporter/nut/tasks/main.yml b/roles/monitoring/prometheus/exporter/nut/tasks/main.yml
index 8245feae..f602472d 100644
--- a/roles/monitoring/prometheus/exporter/nut/tasks/main.yml
+++ b/roles/monitoring/prometheus/exporter/nut/tasks/main.yml
@@ -21,9 +21,11 @@
 - name: register exporter
   copy:
     content: |
-      method: http
-      http:
-        port: 9199
-        path: /ups_metrics
-    dest: /etc/prometheus/exporter/exporter/nut.yml
-  notify: reload prometheus-exporter-exporter
+      location = /nut {
+          proxy_pass http://127.0.0.1:9199/metrics;
+      }
+      location = /nut/ups {
+          proxy_pass http://127.0.0.1:9199/ups_metrics;
+      }
+    dest: /etc/prometheus/exporter/nut.locations
+  notify: reload nginx
diff --git a/roles/monitoring/prometheus/server/templates/jobs/blackbox/https.j2 b/roles/monitoring/prometheus/server/templates/jobs/blackbox/https.j2
index 98a64121..86ff88dd 100644
--- a/roles/monitoring/prometheus/server/templates/jobs/blackbox/https.j2
+++ b/roles/monitoring/prometheus/server/templates/jobs/blackbox/https.j2
@@ -1,8 +1,7 @@
   - job_name: '{{ job }}'
-    metrics_path: /proxy
+    metrics_path: /blackbox
     params:
       module:
-      - blackbox
       - http_tls_2xx
     scheme: https
     tls_config:
diff --git a/roles/monitoring/prometheus/server/templates/jobs/blackbox/ping.j2 b/roles/monitoring/prometheus/server/templates/jobs/blackbox/ping.j2
index 736ffec1..2d3889d2 100644
--- a/roles/monitoring/prometheus/server/templates/jobs/blackbox/ping.j2
+++ b/roles/monitoring/prometheus/server/templates/jobs/blackbox/ping.j2
@@ -1,8 +1,7 @@
   - job_name: '{{ job }}'
-    metrics_path: /proxy
+    metrics_path: /blackbox
     params:
       module:
-      - blackbox
       - icmp
     scheme: https
     tls_config:
diff --git a/roles/monitoring/prometheus/server/templates/jobs/blackbox/ssh.j2 b/roles/monitoring/prometheus/server/templates/jobs/blackbox/ssh.j2
index 166f37ad..97565673 100644
--- a/roles/monitoring/prometheus/server/templates/jobs/blackbox/ssh.j2
+++ b/roles/monitoring/prometheus/server/templates/jobs/blackbox/ssh.j2
@@ -1,8 +1,7 @@
   - job_name: '{{ job }}'
-    metrics_path: /proxy
+    metrics_path: /blackbox
     params:
       module:
-      - blackbox
       - ssh_banner
     scheme: https
     tls_config:
diff --git a/roles/monitoring/prometheus/server/templates/jobs/generic.j2 b/roles/monitoring/prometheus/server/templates/jobs/generic.j2
index b155c5f7..65a95007 100644
--- a/roles/monitoring/prometheus/server/templates/jobs/generic.j2
+++ b/roles/monitoring/prometheus/server/templates/jobs/generic.j2
@@ -1,8 +1,5 @@
   - job_name: '{{ job }}'
-    metrics_path: /proxy
-    params:
-      module:
-      - {{ job }}
+    metrics_path: /{{ job }}
     scheme: https
     tls_config:
       ca_file: /etc/ssl/prometheus/ca-crt.pem
diff --git a/roles/monitoring/prometheus/server/templates/jobs/node.j2 b/roles/monitoring/prometheus/server/templates/jobs/node.j2
index ba9eab31..1b14e1f6 100644
--- a/roles/monitoring/prometheus/server/templates/jobs/node.j2
+++ b/roles/monitoring/prometheus/server/templates/jobs/node.j2
@@ -1,8 +1,5 @@
   - job_name: '{{ job }}'
-    metrics_path: /proxy
-    params:
-      module:
-      - {{ job }}
+    metrics_path: /{{ job }}
     scheme: https
     tls_config:
       ca_file: /etc/ssl/prometheus/ca-crt.pem
diff --git a/roles/monitoring/prometheus/server/templates/jobs/nut/ups.j2 b/roles/monitoring/prometheus/server/templates/jobs/nut/ups.j2
index 3a2c5c62..0cf4ae4e 100644
--- a/roles/monitoring/prometheus/server/templates/jobs/nut/ups.j2
+++ b/roles/monitoring/prometheus/server/templates/jobs/nut/ups.j2
@@ -1,8 +1,5 @@
   - job_name: '{{ job }}'
-    metrics_path: /proxy
-    params:
-      module:
-      - nut
+    metrics_path: /nut/ups
     scheme: https
     tls_config:
       ca_file: /etc/ssl/prometheus/ca-crt.pem
-- 
cgit v1.2.3


From cc89d6d4211aa5aec8e5bef8c854d4929c337887 Mon Sep 17 00:00:00 2001
From: Christian Pointner <equinox@spreadspace.org>
Date: Sun, 26 Sep 2021 03:32:47 +0200
Subject: improved promethues multitarget support

---
 inventory/group_vars/chaos-at-home-ups/vars.yml    |  6 +-
 inventory/group_vars/ele-ups/vars.yml              | 10 +--
 .../group_vars/promzone-chaos-at-home/vars.yml     |  6 +-
 .../group_vars/promzone-elevate-festival/vars.yml  |  1 +
 inventory/host_vars/ch-mon.yml                     | 39 ++++++------
 .../prometheus/exporter/blackbox/tasks/main.yml    |  3 +
 .../prometheus/server/defaults/main/main.yml       |  4 +-
 .../server/defaults/main/rules_blackbox.yml        | 46 +-------------
 .../server/defaults/main/rules_blackbox__https.yml | 20 ------
 .../server/defaults/main/rules_blackbox__ping.yml  | 11 ----
 .../server/defaults/main/rules_blackbox__probe.yml | 74 ++++++++++++++++++++++
 .../server/defaults/main/rules_blackbox__ssh.yml   |  3 -
 .../prometheus/server/filter_plugins/prometheus.py | 10 +--
 roles/monitoring/prometheus/server/tasks/main.yml  |  2 +-
 .../server/templates/jobs/blackbox/https.j2        | 13 ----
 .../server/templates/jobs/blackbox/ping.j2         | 13 ----
 .../server/templates/jobs/blackbox/ssh.j2          | 13 ----
 .../prometheus/server/templates/jobs/nut/ups.j2    | 10 ---
 .../server/templates/targets/blackbox/https.yml.j2 |  4 --
 .../server/templates/targets/blackbox/ping.yml.j2  |  4 --
 .../server/templates/targets/blackbox/probe.yml.j2 |  5 ++
 .../server/templates/targets/blackbox/ssh.yml.j2   |  4 --
 .../server/templates/targets/nut/ups.yml.j2        |  2 +-
 23 files changed, 123 insertions(+), 180 deletions(-)
 delete mode 100644 roles/monitoring/prometheus/server/defaults/main/rules_blackbox__https.yml
 delete mode 100644 roles/monitoring/prometheus/server/defaults/main/rules_blackbox__ping.yml
 create mode 100644 roles/monitoring/prometheus/server/defaults/main/rules_blackbox__probe.yml
 delete mode 100644 roles/monitoring/prometheus/server/defaults/main/rules_blackbox__ssh.yml
 delete mode 100644 roles/monitoring/prometheus/server/templates/jobs/blackbox/https.j2
 delete mode 100644 roles/monitoring/prometheus/server/templates/jobs/blackbox/ping.j2
 delete mode 100644 roles/monitoring/prometheus/server/templates/jobs/blackbox/ssh.j2
 delete mode 100644 roles/monitoring/prometheus/server/templates/jobs/nut/ups.j2
 delete mode 100644 roles/monitoring/prometheus/server/templates/targets/blackbox/https.yml.j2
 delete mode 100644 roles/monitoring/prometheus/server/templates/targets/blackbox/ping.yml.j2
 create mode 100644 roles/monitoring/prometheus/server/templates/targets/blackbox/probe.yml.j2
 delete mode 100644 roles/monitoring/prometheus/server/templates/targets/blackbox/ssh.yml.j2

(limited to 'inventory/host_vars')

diff --git a/inventory/group_vars/chaos-at-home-ups/vars.yml b/inventory/group_vars/chaos-at-home-ups/vars.yml
index f8c1bdf1..7b60e893 100644
--- a/inventory/group_vars/chaos-at-home-ups/vars.yml
+++ b/inventory/group_vars/chaos-at-home-ups/vars.yml
@@ -11,8 +11,8 @@ prometheus_scrape_endpoint: "{{ network_mgmt_zone.prefix | ipaddr(network_mgmt_z
 prometheus_exporters_default:
   - openwrt
 
-prometheus_job_nut__ups:
-  - exporter_hostname: ch-mon
-    instance: "ups-{{ ups_name }}"
+prometheus_job_multitarget_nut__ups:
+  ch-mon:
+  - instance: "ups-{{ ups_name }}"
     ups: "{{ ups_name }}"
     server: "{{ network_mgmt_zone.prefix | ipaddr(network_mgmt_zone.offsets[inventory_hostname]) | ipaddr('address') }}"
diff --git a/inventory/group_vars/ele-ups/vars.yml b/inventory/group_vars/ele-ups/vars.yml
index 1c4613a3..28a5eaff 100644
--- a/inventory/group_vars/ele-ups/vars.yml
+++ b/inventory/group_vars/ele-ups/vars.yml
@@ -14,8 +14,8 @@ prometheus_scrape_endpoint: "{{ network_mgmt_zone.prefix | ipaddr(network_mgmt_z
 prometheus_exporters_default:
   - openwrt
 
-prometheus_job_nut__ups:
-  exporter_hostname: ele-mon
-  instance: "ups-{{ ups_name }}"
-  ups: "{{ ups_name }}"
-  server: "{{ network_mgmt_zone.prefix | ipaddr(network_mgmt_zone.offsets[inventory_hostname]) | ipaddr('address') }}"
+prometheus_job_multitarget_nut__ups:
+  ele-mon:
+  - instance: "ups-{{ ups_name }}"
+    ups: "{{ ups_name }}"
+    server: "{{ network_mgmt_zone.prefix | ipaddr(network_mgmt_zone.offsets[inventory_hostname]) | ipaddr('address') }}"
diff --git a/inventory/group_vars/promzone-chaos-at-home/vars.yml b/inventory/group_vars/promzone-chaos-at-home/vars.yml
index 84ed1263..529bf3e7 100644
--- a/inventory/group_vars/promzone-chaos-at-home/vars.yml
+++ b/inventory/group_vars/promzone-chaos-at-home/vars.yml
@@ -9,10 +9,10 @@ prometheus_server: ch-mon
 prometheus_server_jobs:
   - node
   - openwrt
+  - nut
   - nut/ups
-  - blackbox/ping
-  - blackbox/https
-  - blackbox/ssh
+  - blackbox
+  - blackbox/probe
 
 prometheus_zone_name: chaos@home
 prometheus_zone_targets: "{{ groups['promzone-chaos-at-home'] }}"
diff --git a/inventory/group_vars/promzone-elevate-festival/vars.yml b/inventory/group_vars/promzone-elevate-festival/vars.yml
index 43115dc4..b3321614 100644
--- a/inventory/group_vars/promzone-elevate-festival/vars.yml
+++ b/inventory/group_vars/promzone-elevate-festival/vars.yml
@@ -9,6 +9,7 @@ prometheus_server: ele-mon
 prometheus_server_jobs:
   - node
   - openwrt
+  - nut
   - nut/ups
 
 prometheus_zone_name: Elevate Festival
diff --git a/inventory/host_vars/ch-mon.yml b/inventory/host_vars/ch-mon.yml
index d1a710b9..b2402d0c 100644
--- a/inventory/host_vars/ch-mon.yml
+++ b/inventory/host_vars/ch-mon.yml
@@ -76,26 +76,25 @@ prometheus_exporter_blackbox_modules_extra:
   icmp:
     prober: icmp
 
-prometheus_job_blackbox__ping:
-  - exporter_hostname: ch-mon
-    instance: "ping-magentagw"
-    address: 62.99.185.129
-  - exporter_hostname: ch-mon
-    instance: "ping-quad9"
-    address: 9.9.9.9
-
-prometheus_job_blackbox__https:
-  - exporter_hostname: ch-mon
-    instance: "https-pan.chaos-at-home.org"
-    address: "https://pan.chaos-at-home.org"
-  - exporter_hostname: ch-mon
-    instance: "https-mimas.chaos-at-home.org"
-    address: "https://mimas.chaos-at-home.org"
-
-prometheus_job_blackbox__ssh:
-  - exporter_hostname: ch-mon
-    instance: "ssh-{{ inventory_hostname }}"
-    address: "{{ network_zones.svc.prefix | ipaddr(network_zones.svc.offsets[inventory_hostname]) | ipaddr('address') }}:{{ ansible_port | default(22) }}"
+prometheus_job_multitarget_blackbox__probe:
+  ch-mon:
+  - instance: "ping-magentagw"
+    target: 62.99.185.129
+    module: icmp
+  - instance: "ping-quad9"
+    target: 9.9.9.9
+    module: icmp
+
+  - instance: "https-pan.chaos-at-home.org"
+    target: "https://pan.chaos-at-home.org"
+    module: http_tls_2xx
+  - instance: "https-mimas.chaos-at-home.org"
+    target: "https://mimas.chaos-at-home.org"
+    module: http_tls_2xx
+
+  - instance: "ssh-{{ inventory_hostname }}"
+    target: "{{ network_zones.svc.prefix | ipaddr(network_zones.svc.offsets[inventory_hostname]) | ipaddr('address') }}:{{ ansible_port | default(22) }}"
+    module: ssh_banner
 
 
 promethues_alertmanager_smtp:
diff --git a/roles/monitoring/prometheus/exporter/blackbox/tasks/main.yml b/roles/monitoring/prometheus/exporter/blackbox/tasks/main.yml
index f9793df6..c4cabfce 100644
--- a/roles/monitoring/prometheus/exporter/blackbox/tasks/main.yml
+++ b/roles/monitoring/prometheus/exporter/blackbox/tasks/main.yml
@@ -33,6 +33,9 @@
   copy:
     content: |
       location = /blackbox {
+          proxy_pass http://127.0.0.1:9115/metrics;
+      }
+      location = /blackbox/probe {
           proxy_pass http://127.0.0.1:9115/probe;
       }
     dest: /etc/prometheus/exporter/blackbox.locations
diff --git a/roles/monitoring/prometheus/server/defaults/main/main.yml b/roles/monitoring/prometheus/server/defaults/main/main.yml
index 09cd150c..7781fd69 100644
--- a/roles/monitoring/prometheus/server/defaults/main/main.yml
+++ b/roles/monitoring/prometheus/server/defaults/main/main.yml
@@ -17,9 +17,7 @@ prometheus_server_rules:
   nut: "{{ prometheus_server_rules_nut + prometheus_server_rules_nut_extra }}"
   nut/ups: "{{ prometheus_server_rules_nut__ups + prometheus_server_rules_nut__ups_extra }}"
   blackbox: "{{ prometheus_server_rules_blackbox + prometheus_server_rules_blackbox_extra }}"
-  blackbox/ping: "{{ prometheus_server_rules_blackbox__ping + prometheus_server_rules_blackbox__ping_extra }}"
-  blackbox/https: "{{ prometheus_server_rules_blackbox__https + prometheus_server_rules_blackbox__https_extra }}"
-  blackbox/ssh: "{{ prometheus_server_rules_blackbox__ssh + prometheus_server_rules_blackbox__ssh_extra }}"
+  blackbox/probe: "{{ prometheus_server_rules_blackbox__probe + prometheus_server_rules_blackbox__probe_extra }}"
 
 # prometheus_server_alertmanager:
 #   url: "127.0.0.1:9093"
diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_blackbox.yml b/roles/monitoring/prometheus/server/defaults/main/rules_blackbox.yml
index d5c1fd42..99f2e83c 100644
--- a/roles/monitoring/prometheus/server/defaults/main/rules_blackbox.yml
+++ b/roles/monitoring/prometheus/server/defaults/main/rules_blackbox.yml
@@ -1,47 +1,3 @@
 ---
 prometheus_server_rules_blackbox_extra: []
-prometheus_server_rules_blackbox:
-  - alert: BlackboxProbeFailed
-    expr: probe_success == 0
-    for: 0m
-    labels:
-      severity: critical
-    annotations:
-      summary: Blackbox probe failed (instance {{ '{{' }} $labels.instance {{ '}}' }})
-      description: "Probe failed\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
-
-  - alert: BlackboxSlowProbe
-    expr: avg_over_time(probe_duration_seconds[1m]) > 1
-    for: 1m
-    labels:
-      severity: warning
-    annotations:
-      summary: Blackbox slow probe (instance {{ '{{' }} $labels.instance {{ '}}' }})
-      description: "Blackbox probe took more than 1s to complete\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
-
-  - alert: BlackboxSslCertificateWillExpireSoon
-    expr: probe_ssl_earliest_cert_expiry - time() < 86400 * 30
-    for: 0m
-    labels:
-      severity: warning
-    annotations:
-      summary: Blackbox SSL certificate will expire soon (instance {{ '{{' }} $labels.instance {{ '}}' }})
-      description: "SSL certificate expires in 30 days\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
-
-  - alert: BlackboxSslCertificateWillExpireSoon
-    expr: probe_ssl_earliest_cert_expiry - time() < 86400 * 3
-    for: 0m
-    labels:
-      severity: critical
-    annotations:
-      summary: Blackbox SSL certificate will expire soon (instance {{ '{{' }} $labels.instance {{ '}}' }})
-      description: "SSL certificate expires in 3 days\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
-
-  - alert: BlackboxSslCertificateExpired
-    expr: probe_ssl_earliest_cert_expiry - time() <= 0
-    for: 0m
-    labels:
-      severity: critical
-    annotations:
-      summary: Blackbox SSL certificate expired (instance {{ '{{' }} $labels.instance {{ '}}' }})
-      description: "SSL certificate has expired already\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+prometheus_server_rules_blackbox: []
diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_blackbox__https.yml b/roles/monitoring/prometheus/server/defaults/main/rules_blackbox__https.yml
deleted file mode 100644
index 140e3b4f..00000000
--- a/roles/monitoring/prometheus/server/defaults/main/rules_blackbox__https.yml
+++ /dev/null
@@ -1,20 +0,0 @@
----
-prometheus_server_rules_blackbox__https_extra: []
-prometheus_server_rules_blackbox__https:
-  - alert: BlackboxProbeHttpFailure
-    expr: probe_http_status_code <= 199 OR probe_http_status_code >= 400
-    for: 0m
-    labels:
-      severity: critical
-    annotations:
-      summary: Blackbox probe HTTP failure (instance {{ '{{' }} $labels.instance {{ '}}' }})
-      description: "HTTP status code is not 200-399\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
-
-  - alert: BlackboxProbeSlowHttp
-    expr: avg_over_time(probe_http_duration_seconds[1m]) > 1
-    for: 1m
-    labels:
-      severity: warning
-    annotations:
-      summary: Blackbox probe slow HTTP (instance {{ '{{' }} $labels.instance {{ '}}' }})
-      description: "HTTP request took more than 1s\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_blackbox__ping.yml b/roles/monitoring/prometheus/server/defaults/main/rules_blackbox__ping.yml
deleted file mode 100644
index cc87b6b1..00000000
--- a/roles/monitoring/prometheus/server/defaults/main/rules_blackbox__ping.yml
+++ /dev/null
@@ -1,11 +0,0 @@
----
-prometheus_server_rules_blackbox__ping_extra: []
-prometheus_server_rules_blackbox__ping:
-  - alert: BlackboxProbeSlowPing
-    expr: avg_over_time(probe_icmp_duration_seconds[1m]) > 1
-    for: 1m
-    labels:
-      severity: warning
-    annotations:
-      summary: Blackbox probe slow ping (instance {{ '{{' }} $labels.instance {{ '}}' }})
-      description: "Blackbox ping took more than 1s\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_blackbox__probe.yml b/roles/monitoring/prometheus/server/defaults/main/rules_blackbox__probe.yml
new file mode 100644
index 00000000..9f9d2292
--- /dev/null
+++ b/roles/monitoring/prometheus/server/defaults/main/rules_blackbox__probe.yml
@@ -0,0 +1,74 @@
+---
+prometheus_server_rules_blackbox__probe_extra: []
+prometheus_server_rules_blackbox__probe:
+  - alert: BlackboxProbeFailed
+    expr: probe_success == 0
+    for: 0m
+    labels:
+      severity: critical
+    annotations:
+      summary: Blackbox probe failed (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "Probe failed\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: BlackboxSlowProbe
+    expr: avg_over_time(probe_duration_seconds[1m]) > 1
+    for: 1m
+    labels:
+      severity: warning
+    annotations:
+      summary: Blackbox slow probe (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "Blackbox probe took more than 1s to complete\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: BlackboxSslCertificateWillExpireSoon
+    expr: probe_ssl_earliest_cert_expiry - time() < 86400 * 30
+    for: 0m
+    labels:
+      severity: warning
+    annotations:
+      summary: Blackbox SSL certificate will expire soon (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "SSL certificate expires in 30 days\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: BlackboxSslCertificateWillExpireSoon
+    expr: probe_ssl_earliest_cert_expiry - time() < 86400 * 3
+    for: 0m
+    labels:
+      severity: critical
+    annotations:
+      summary: Blackbox SSL certificate will expire soon (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "SSL certificate expires in 3 days\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: BlackboxSslCertificateExpired
+    expr: probe_ssl_earliest_cert_expiry - time() <= 0
+    for: 0m
+    labels:
+      severity: critical
+    annotations:
+      summary: Blackbox SSL certificate expired (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "SSL certificate has expired already\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: BlackboxProbeHttpFailure
+    expr: probe_http_status_code <= 199 OR probe_http_status_code >= 400
+    for: 0m
+    labels:
+      severity: critical
+    annotations:
+      summary: Blackbox probe HTTP failure (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "HTTP status code is not 200-399\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: BlackboxProbeSlowHttp
+    expr: avg_over_time(probe_http_duration_seconds[1m]) > 1
+    for: 1m
+    labels:
+      severity: warning
+    annotations:
+      summary: Blackbox probe slow HTTP (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "HTTP request took more than 1s\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: BlackboxProbeSlowPing
+    expr: avg_over_time(probe_icmp_duration_seconds[1m]) > 1
+    for: 1m
+    labels:
+      severity: warning
+    annotations:
+      summary: Blackbox probe slow ping (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "Blackbox ping took more than 1s\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_blackbox__ssh.yml b/roles/monitoring/prometheus/server/defaults/main/rules_blackbox__ssh.yml
deleted file mode 100644
index 8e717c41..00000000
--- a/roles/monitoring/prometheus/server/defaults/main/rules_blackbox__ssh.yml
+++ /dev/null
@@ -1,3 +0,0 @@
----
-prometheus_server_rules_blackbox__ssh_extra: []
-prometheus_server_rules_blackbox__ssh: []
diff --git a/roles/monitoring/prometheus/server/filter_plugins/prometheus.py b/roles/monitoring/prometheus/server/filter_plugins/prometheus.py
index 1443e837..d91ef619 100644
--- a/roles/monitoring/prometheus/server/filter_plugins/prometheus.py
+++ b/roles/monitoring/prometheus/server/filter_plugins/prometheus.py
@@ -11,10 +11,12 @@ def prometheus_job_targets(hostvars, jobs, targets):
         result = []
         for job in jobs:
             for target in targets:
-                special_config_varname = 'prometheus_job_' + job.replace('-', '_').replace('/', '__')
-                if special_config_varname in hostvars[target]:
-                    for config in hostvars[target][special_config_varname]:
-                        result.append({'job': job, 'instance': config['instance'], 'config': config, 'enabled': True})
+                multitarget_config_varname = 'prometheus_job_multitarget_' + job.replace('-', '_').replace('/', '__')
+                if multitarget_config_varname in hostvars[target]:
+                    for exporter_hostname, configs in hostvars[target][multitarget_config_varname].items():
+                        for config in configs:
+                            result.append({'job': job, 'instance': config['instance'], 'enabled': True,
+                                           'exporter_hostname': exporter_hostname, 'config': config})
 
                 else:
                     enabled = job in hostvars[target]['prometheus_exporters_default'] or job in hostvars[target]['prometheus_exporters_extra']
diff --git a/roles/monitoring/prometheus/server/tasks/main.yml b/roles/monitoring/prometheus/server/tasks/main.yml
index c0928cc3..16167c9c 100644
--- a/roles/monitoring/prometheus/server/tasks/main.yml
+++ b/roles/monitoring/prometheus/server/tasks/main.yml
@@ -83,7 +83,7 @@
     state: directory
 
 - name: generate rules files for all jobs
-  loop: "{{ prometheus_server_jobs | union(['prometheus']) | union(prometheus_server_jobs | select('match', '.*/.*') | map('dirname') | unique) }}"
+  loop: "{{ prometheus_server_jobs | union(['prometheus']) }}"
   template:
     src: rules.yml.j2
     dest: "/etc/prometheus/rules/{{ item }}.yml"
diff --git a/roles/monitoring/prometheus/server/templates/jobs/blackbox/https.j2 b/roles/monitoring/prometheus/server/templates/jobs/blackbox/https.j2
deleted file mode 100644
index 86ff88dd..00000000
--- a/roles/monitoring/prometheus/server/templates/jobs/blackbox/https.j2
+++ /dev/null
@@ -1,13 +0,0 @@
-  - job_name: '{{ job }}'
-    metrics_path: /blackbox
-    params:
-      module:
-      - http_tls_2xx
-    scheme: https
-    tls_config:
-      ca_file: /etc/ssl/prometheus/ca-crt.pem
-      cert_file: /etc/ssl/prometheus/server/scrape-crt.pem
-      key_file: /etc/ssl/prometheus/server/scrape-key.pem
-    file_sd_configs:
-    - files:
-      - "/etc/prometheus/targets/{{ job }}/*.yml"
diff --git a/roles/monitoring/prometheus/server/templates/jobs/blackbox/ping.j2 b/roles/monitoring/prometheus/server/templates/jobs/blackbox/ping.j2
deleted file mode 100644
index 2d3889d2..00000000
--- a/roles/monitoring/prometheus/server/templates/jobs/blackbox/ping.j2
+++ /dev/null
@@ -1,13 +0,0 @@
-  - job_name: '{{ job }}'
-    metrics_path: /blackbox
-    params:
-      module:
-      - icmp
-    scheme: https
-    tls_config:
-      ca_file: /etc/ssl/prometheus/ca-crt.pem
-      cert_file: /etc/ssl/prometheus/server/scrape-crt.pem
-      key_file: /etc/ssl/prometheus/server/scrape-key.pem
-    file_sd_configs:
-    - files:
-      - "/etc/prometheus/targets/{{ job }}/*.yml"
diff --git a/roles/monitoring/prometheus/server/templates/jobs/blackbox/ssh.j2 b/roles/monitoring/prometheus/server/templates/jobs/blackbox/ssh.j2
deleted file mode 100644
index 97565673..00000000
--- a/roles/monitoring/prometheus/server/templates/jobs/blackbox/ssh.j2
+++ /dev/null
@@ -1,13 +0,0 @@
-  - job_name: '{{ job }}'
-    metrics_path: /blackbox
-    params:
-      module:
-      - ssh_banner
-    scheme: https
-    tls_config:
-      ca_file: /etc/ssl/prometheus/ca-crt.pem
-      cert_file: /etc/ssl/prometheus/server/scrape-crt.pem
-      key_file: /etc/ssl/prometheus/server/scrape-key.pem
-    file_sd_configs:
-    - files:
-      - "/etc/prometheus/targets/{{ job }}/*.yml"
diff --git a/roles/monitoring/prometheus/server/templates/jobs/nut/ups.j2 b/roles/monitoring/prometheus/server/templates/jobs/nut/ups.j2
deleted file mode 100644
index 0cf4ae4e..00000000
--- a/roles/monitoring/prometheus/server/templates/jobs/nut/ups.j2
+++ /dev/null
@@ -1,10 +0,0 @@
-  - job_name: '{{ job }}'
-    metrics_path: /nut/ups
-    scheme: https
-    tls_config:
-      ca_file: /etc/ssl/prometheus/ca-crt.pem
-      cert_file: /etc/ssl/prometheus/server/scrape-crt.pem
-      key_file: /etc/ssl/prometheus/server/scrape-key.pem
-    file_sd_configs:
-    - files:
-      - "/etc/prometheus/targets/{{ job }}/*.yml"
diff --git a/roles/monitoring/prometheus/server/templates/targets/blackbox/https.yml.j2 b/roles/monitoring/prometheus/server/templates/targets/blackbox/https.yml.j2
deleted file mode 100644
index 29c89590..00000000
--- a/roles/monitoring/prometheus/server/templates/targets/blackbox/https.yml.j2
+++ /dev/null
@@ -1,4 +0,0 @@
-- targets: [ '{{ hostvars[target.config.exporter_hostname].prometheus_scrape_endpoint }}' ]
-  labels:
-    instance: '{{ target.instance }}'
-    __param_target: '{{ target.config.address }}'
diff --git a/roles/monitoring/prometheus/server/templates/targets/blackbox/ping.yml.j2 b/roles/monitoring/prometheus/server/templates/targets/blackbox/ping.yml.j2
deleted file mode 100644
index 29c89590..00000000
--- a/roles/monitoring/prometheus/server/templates/targets/blackbox/ping.yml.j2
+++ /dev/null
@@ -1,4 +0,0 @@
-- targets: [ '{{ hostvars[target.config.exporter_hostname].prometheus_scrape_endpoint }}' ]
-  labels:
-    instance: '{{ target.instance }}'
-    __param_target: '{{ target.config.address }}'
diff --git a/roles/monitoring/prometheus/server/templates/targets/blackbox/probe.yml.j2 b/roles/monitoring/prometheus/server/templates/targets/blackbox/probe.yml.j2
new file mode 100644
index 00000000..4e336873
--- /dev/null
+++ b/roles/monitoring/prometheus/server/templates/targets/blackbox/probe.yml.j2
@@ -0,0 +1,5 @@
+- targets: [ '{{ hostvars[target.exporter_hostname].prometheus_scrape_endpoint }}' ]
+  labels:
+    instance: '{{ target.instance }}'
+    __param_target: '{{ target.config.target }}'
+    __param_module: '{{ target.config.module }}'
diff --git a/roles/monitoring/prometheus/server/templates/targets/blackbox/ssh.yml.j2 b/roles/monitoring/prometheus/server/templates/targets/blackbox/ssh.yml.j2
deleted file mode 100644
index 29c89590..00000000
--- a/roles/monitoring/prometheus/server/templates/targets/blackbox/ssh.yml.j2
+++ /dev/null
@@ -1,4 +0,0 @@
-- targets: [ '{{ hostvars[target.config.exporter_hostname].prometheus_scrape_endpoint }}' ]
-  labels:
-    instance: '{{ target.instance }}'
-    __param_target: '{{ target.config.address }}'
diff --git a/roles/monitoring/prometheus/server/templates/targets/nut/ups.yml.j2 b/roles/monitoring/prometheus/server/templates/targets/nut/ups.yml.j2
index 6003cd46..c60077c7 100644
--- a/roles/monitoring/prometheus/server/templates/targets/nut/ups.yml.j2
+++ b/roles/monitoring/prometheus/server/templates/targets/nut/ups.yml.j2
@@ -1,4 +1,4 @@
-- targets: [ '{{ hostvars[target.config.exporter_hostname].prometheus_scrape_endpoint }}' ]
+- targets: [ '{{ hostvars[target.exporter_hostname].prometheus_scrape_endpoint }}' ]
   labels:
     instance: '{{ target.instance }}'
     __param_ups: '{{ target.config.ups }}'
-- 
cgit v1.2.3


From 419ede2858769e4414a23a42b57931b83cf70d8c Mon Sep 17 00:00:00 2001
From: Christian Pointner <equinox@spreadspace.org>
Date: Sun, 26 Sep 2021 04:08:58 +0200
Subject: add job configs for ipmi and ipmi/remote

---
 inventory/host_vars/ch-testvm-prometheus.yml                         | 3 ---
 roles/monitoring/prometheus/exporter/ipmi/defaults/main.yml          | 3 ++-
 roles/monitoring/prometheus/server/defaults/main/main.yml            | 2 ++
 roles/monitoring/prometheus/server/defaults/main/rules_ipmi.yml      | 4 ++++
 .../prometheus/server/defaults/main/rules_ipmi__remote.yml           | 4 ++++
 .../prometheus/server/templates/targets/ipmi/remote.yml.j2           | 5 +++++
 6 files changed, 17 insertions(+), 4 deletions(-)
 create mode 100644 roles/monitoring/prometheus/server/defaults/main/rules_ipmi.yml
 create mode 100644 roles/monitoring/prometheus/server/defaults/main/rules_ipmi__remote.yml
 create mode 100644 roles/monitoring/prometheus/server/templates/targets/ipmi/remote.yml.j2

(limited to 'inventory/host_vars')

diff --git a/inventory/host_vars/ch-testvm-prometheus.yml b/inventory/host_vars/ch-testvm-prometheus.yml
index 939fa398..e539735f 100644
--- a/inventory/host_vars/ch-testvm-prometheus.yml
+++ b/inventory/host_vars/ch-testvm-prometheus.yml
@@ -36,9 +36,6 @@ network:
 spreadspace_apt_repo_components:
   - prometheus
 
-prometheus_exporters_extra:
-  - ipmi
-
 
 containerd_storage:
   type: lvm
diff --git a/roles/monitoring/prometheus/exporter/ipmi/defaults/main.yml b/roles/monitoring/prometheus/exporter/ipmi/defaults/main.yml
index 9b99f9a5..6cf14f76 100644
--- a/roles/monitoring/prometheus/exporter/ipmi/defaults/main.yml
+++ b/roles/monitoring/prometheus/exporter/ipmi/defaults/main.yml
@@ -1,6 +1,7 @@
 ---
 prometheus_exporter_ipmi_modules:
-  default: {}
+  default:
+    collectors: []
   #   collectors:
   #   - bmc
   #   - ipmi
diff --git a/roles/monitoring/prometheus/server/defaults/main/main.yml b/roles/monitoring/prometheus/server/defaults/main/main.yml
index 7781fd69..1e0ccf78 100644
--- a/roles/monitoring/prometheus/server/defaults/main/main.yml
+++ b/roles/monitoring/prometheus/server/defaults/main/main.yml
@@ -18,6 +18,8 @@ prometheus_server_rules:
   nut/ups: "{{ prometheus_server_rules_nut__ups + prometheus_server_rules_nut__ups_extra }}"
   blackbox: "{{ prometheus_server_rules_blackbox + prometheus_server_rules_blackbox_extra }}"
   blackbox/probe: "{{ prometheus_server_rules_blackbox__probe + prometheus_server_rules_blackbox__probe_extra }}"
+  ipmi: "{{ prometheus_server_rules_ipmi + prometheus_server_rules_ipmi_extra }}"
+  ipmi/remote: "{{ prometheus_server_rules_ipmi__remote + prometheus_server_rules_ipmi__remote_extra }}"
 
 # prometheus_server_alertmanager:
 #   url: "127.0.0.1:9093"
diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_ipmi.yml b/roles/monitoring/prometheus/server/defaults/main/rules_ipmi.yml
new file mode 100644
index 00000000..41dcd7e9
--- /dev/null
+++ b/roles/monitoring/prometheus/server/defaults/main/rules_ipmi.yml
@@ -0,0 +1,4 @@
+---
+prometheus_server_rules_ipmi_extra: []
+prometheus_server_rules_ipmi: []
+## TODO: add common IPMI alert rules
diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_ipmi__remote.yml b/roles/monitoring/prometheus/server/defaults/main/rules_ipmi__remote.yml
new file mode 100644
index 00000000..1f9338ea
--- /dev/null
+++ b/roles/monitoring/prometheus/server/defaults/main/rules_ipmi__remote.yml
@@ -0,0 +1,4 @@
+---
+prometheus_server_rules_ipmi__remote_extra: []
+prometheus_server_rules_ipmi__remote: []
+## TODO: add remote-IPMI specific alert rules
diff --git a/roles/monitoring/prometheus/server/templates/targets/ipmi/remote.yml.j2 b/roles/monitoring/prometheus/server/templates/targets/ipmi/remote.yml.j2
new file mode 100644
index 00000000..4e336873
--- /dev/null
+++ b/roles/monitoring/prometheus/server/templates/targets/ipmi/remote.yml.j2
@@ -0,0 +1,5 @@
+- targets: [ '{{ hostvars[target.exporter_hostname].prometheus_scrape_endpoint }}' ]
+  labels:
+    instance: '{{ target.instance }}'
+    __param_target: '{{ target.config.target }}'
+    __param_module: '{{ target.config.module }}'
-- 
cgit v1.2.3