From a8e8cb2ed3d5e68d89edd8785ed59f0ee45f81bf Mon Sep 17 00:00:00 2001
From: Christian Pointner <equinox@spreadspace.org>
Date: Tue, 21 Sep 2021 19:34:25 +0200
Subject: prometheus: simplify job config

---
 roles/monitoring/prometheus/exporter/blackbox/tasks/main.yml | 1 +
 1 file changed, 1 insertion(+)

(limited to 'roles/monitoring/prometheus/exporter/blackbox/tasks')

diff --git a/roles/monitoring/prometheus/exporter/blackbox/tasks/main.yml b/roles/monitoring/prometheus/exporter/blackbox/tasks/main.yml
index 7ecd8113..96c247ec 100644
--- a/roles/monitoring/prometheus/exporter/blackbox/tasks/main.yml
+++ b/roles/monitoring/prometheus/exporter/blackbox/tasks/main.yml
@@ -1,4 +1,5 @@
 ---
+  ## TODO: pin version
 - name: install apt packages
   apt:
     name: prom-exporter-blackbox
-- 
cgit v1.2.3


From d4a5276b2813f95d56e8fadb0e6d8ff169b8eecb Mon Sep 17 00:00:00 2001
From: Christian Pointner <equinox@spreadspace.org>
Date: Thu, 23 Sep 2021 14:40:38 +0200
Subject: move exporter_exporter configuration into canonical config dir

---
 roles/monitoring/prometheus/exporter/base/tasks/main.yml       | 2 +-
 roles/monitoring/prometheus/exporter/base/templates/service.j2 | 2 +-
 roles/monitoring/prometheus/exporter/blackbox/tasks/main.yml   | 2 +-
 roles/monitoring/prometheus/exporter/mikrotik/tasks/main.yml   | 2 +-
 roles/monitoring/prometheus/exporter/node/tasks/main.yml       | 2 +-
 roles/monitoring/prometheus/exporter/nut/tasks/main.yml        | 2 +-
 6 files changed, 6 insertions(+), 6 deletions(-)

(limited to 'roles/monitoring/prometheus/exporter/blackbox/tasks')

diff --git a/roles/monitoring/prometheus/exporter/base/tasks/main.yml b/roles/monitoring/prometheus/exporter/base/tasks/main.yml
index 9e35c48d..eeb2a23d 100644
--- a/roles/monitoring/prometheus/exporter/base/tasks/main.yml
+++ b/roles/monitoring/prometheus/exporter/base/tasks/main.yml
@@ -14,7 +14,7 @@
 
 - name: create configuration directories
   file:
-    path: /etc/prometheus/exporter/enabled
+    path: /etc/prometheus/exporter/exporter
     state: directory
 
 - name: add user for prometheus-exporter
diff --git a/roles/monitoring/prometheus/exporter/base/templates/service.j2 b/roles/monitoring/prometheus/exporter/base/templates/service.j2
index c24baf43..3d44744a 100644
--- a/roles/monitoring/prometheus/exporter/base/templates/service.j2
+++ b/roles/monitoring/prometheus/exporter/base/templates/service.j2
@@ -4,7 +4,7 @@ Description=Prometheus exporter proxy
 [Service]
 Restart=always
 User=prometheus-exporter
-ExecStart=/usr/bin/prometheus-exporter-exporter -config.dirs=/etc/prometheus/exporter/enabled -config.file="" -web.listen-address="" -web.tls.listen-address="{{ prometheus_exporter_listen }}" -web.tls.cert="/etc/ssl/prometheus/exporter/crt.pem" -web.tls.key="/etc/ssl/prometheus/exporter/key.pem" --web.tls.ca="/etc/ssl/prometheus/ca-crt.pem" -web.tls.verify
+ExecStart=/usr/bin/prometheus-exporter-exporter -config.dirs=/etc/prometheus/exporter/exporter -config.file="" -web.listen-address="" -web.tls.listen-address="{{ prometheus_exporter_listen }}" -web.tls.cert="/etc/ssl/prometheus/exporter/crt.pem" -web.tls.key="/etc/ssl/prometheus/exporter/key.pem" --web.tls.ca="/etc/ssl/prometheus/ca-crt.pem" -web.tls.verify
 {# TODO: implement reloading once the exporter_exporter supports this #}
 
 # systemd hardening-options
diff --git a/roles/monitoring/prometheus/exporter/blackbox/tasks/main.yml b/roles/monitoring/prometheus/exporter/blackbox/tasks/main.yml
index 96c247ec..cab521cc 100644
--- a/roles/monitoring/prometheus/exporter/blackbox/tasks/main.yml
+++ b/roles/monitoring/prometheus/exporter/blackbox/tasks/main.yml
@@ -36,5 +36,5 @@
       http:
         port: 9115
         path: '/probe'
-    dest: /etc/prometheus/exporter/enabled/blackbox.yml
+    dest: /etc/prometheus/exporter/exporter/blackbox.yml
   notify: reload prometheus-exporter-exporter
diff --git a/roles/monitoring/prometheus/exporter/mikrotik/tasks/main.yml b/roles/monitoring/prometheus/exporter/mikrotik/tasks/main.yml
index dda33e9f..07219c68 100644
--- a/roles/monitoring/prometheus/exporter/mikrotik/tasks/main.yml
+++ b/roles/monitoring/prometheus/exporter/mikrotik/tasks/main.yml
@@ -38,5 +38,5 @@
       method: http
       http:
         port: 9436
-    dest: /etc/prometheus/exporter/enabled/mikrotik.yml
+    dest: /etc/prometheus/exporter/exporter/mikrotik.yml
   notify: reload prometheus-exporter-exporter
diff --git a/roles/monitoring/prometheus/exporter/node/tasks/main.yml b/roles/monitoring/prometheus/exporter/node/tasks/main.yml
index 00a4ab3f..3fa0a1ec 100644
--- a/roles/monitoring/prometheus/exporter/node/tasks/main.yml
+++ b/roles/monitoring/prometheus/exporter/node/tasks/main.yml
@@ -29,5 +29,5 @@
       method: http
       http:
         port: 9100
-    dest: /etc/prometheus/exporter/enabled/node.yml
+    dest: /etc/prometheus/exporter/exporter/node.yml
   notify: reload prometheus-exporter-exporter
diff --git a/roles/monitoring/prometheus/exporter/nut/tasks/main.yml b/roles/monitoring/prometheus/exporter/nut/tasks/main.yml
index 78a8e817..8245feae 100644
--- a/roles/monitoring/prometheus/exporter/nut/tasks/main.yml
+++ b/roles/monitoring/prometheus/exporter/nut/tasks/main.yml
@@ -25,5 +25,5 @@
       http:
         port: 9199
         path: /ups_metrics
-    dest: /etc/prometheus/exporter/enabled/nut.yml
+    dest: /etc/prometheus/exporter/exporter/nut.yml
   notify: reload prometheus-exporter-exporter
-- 
cgit v1.2.3


From 2316917055ec9399966033cc4944f5e5662c0136 Mon Sep 17 00:00:00 2001
From: Christian Pointner <equinox@spreadspace.org>
Date: Fri, 24 Sep 2021 22:33:33 +0200
Subject: add prometheus exporter IPMI

---
 roles/monitoring/prometheus/exporter/TODO          |  4 --
 .../prometheus/exporter/blackbox/tasks/main.yml    |  2 +-
 .../prometheus/exporter/ipmi/defaults/main.yml     | 25 ++++++++++++
 .../prometheus/exporter/ipmi/handlers/main.yml     | 16 ++++++++
 .../prometheus/exporter/ipmi/tasks/main.yml        | 47 ++++++++++++++++++++++
 .../exporter/ipmi/templates/config.yml.j2          |  4 ++
 .../prometheus/exporter/ipmi/templates/service.j2  | 32 +++++++++++++++
 7 files changed, 125 insertions(+), 5 deletions(-)
 create mode 100644 roles/monitoring/prometheus/exporter/ipmi/defaults/main.yml
 create mode 100644 roles/monitoring/prometheus/exporter/ipmi/handlers/main.yml
 create mode 100644 roles/monitoring/prometheus/exporter/ipmi/tasks/main.yml
 create mode 100644 roles/monitoring/prometheus/exporter/ipmi/templates/config.yml.j2
 create mode 100644 roles/monitoring/prometheus/exporter/ipmi/templates/service.j2

(limited to 'roles/monitoring/prometheus/exporter/blackbox/tasks')

diff --git a/roles/monitoring/prometheus/exporter/TODO b/roles/monitoring/prometheus/exporter/TODO
index 79ff8721..57179464 100644
--- a/roles/monitoring/prometheus/exporter/TODO
+++ b/roles/monitoring/prometheus/exporter/TODO
@@ -1,7 +1,3 @@
-IPMI Exporter:
-  - https://github.com/soundcloud/ipmi_exporter
-  - https://packages.debian.org/bullseye/prometheus-ipmi-exporter
-
 Postfix Exporter:
   - https://github.com/kumina/postfix_exporter
   - https://packages.debian.org/bullseye/prometheus-postfix-exporter
diff --git a/roles/monitoring/prometheus/exporter/blackbox/tasks/main.yml b/roles/monitoring/prometheus/exporter/blackbox/tasks/main.yml
index cab521cc..782c3561 100644
--- a/roles/monitoring/prometheus/exporter/blackbox/tasks/main.yml
+++ b/roles/monitoring/prometheus/exporter/blackbox/tasks/main.yml
@@ -35,6 +35,6 @@
       method: http
       http:
         port: 9115
-        path: '/probe'
+        path: /probe
     dest: /etc/prometheus/exporter/exporter/blackbox.yml
   notify: reload prometheus-exporter-exporter
diff --git a/roles/monitoring/prometheus/exporter/ipmi/defaults/main.yml b/roles/monitoring/prometheus/exporter/ipmi/defaults/main.yml
new file mode 100644
index 00000000..9b99f9a5
--- /dev/null
+++ b/roles/monitoring/prometheus/exporter/ipmi/defaults/main.yml
@@ -0,0 +1,25 @@
+---
+prometheus_exporter_ipmi_modules:
+  default: {}
+  #   collectors:
+  #   - bmc
+  #   - ipmi
+  #   - chassis
+  #   - dcmi
+  #   - sel
+  #   - sm-lan-mode
+  #   exclude_sensor_ids:
+  #   - 2
+  #   - 29
+  #   - 32
+  # thatspecialhost:
+  #   user: "some_user"
+  #   pass: "secret_pw"
+  #   privilege: "admin"
+  #   driver: "LAN"
+  #   collectors:
+  #   - ipmi
+  #   - sel
+  #   custom_args:
+  #     ipmi:
+  #     - "--bridge-sensors"
diff --git a/roles/monitoring/prometheus/exporter/ipmi/handlers/main.yml b/roles/monitoring/prometheus/exporter/ipmi/handlers/main.yml
new file mode 100644
index 00000000..40a945ae
--- /dev/null
+++ b/roles/monitoring/prometheus/exporter/ipmi/handlers/main.yml
@@ -0,0 +1,16 @@
+---
+- name: restart prometheus-ipmi-exporter
+  service:
+    name: prometheus-ipmi-exporter
+    state: restarted
+
+- name: reload prometheus-ipmi-exporter
+  service:
+    name: prometheus-ipmi-exporter
+    state: reloaded
+
+- name: reload prometheus-exporter-exporter
+  service:
+    name: prometheus-exporter-exporter
+    ## TODO: implement reload once exporter_exporter supports this...
+    state: restarted
diff --git a/roles/monitoring/prometheus/exporter/ipmi/tasks/main.yml b/roles/monitoring/prometheus/exporter/ipmi/tasks/main.yml
new file mode 100644
index 00000000..9e63f692
--- /dev/null
+++ b/roles/monitoring/prometheus/exporter/ipmi/tasks/main.yml
@@ -0,0 +1,47 @@
+---
+  ## TODO: pin version
+- name: install apt packages
+  apt:
+    name: prom-exporter-ipmi
+    state: present
+
+- name: create config directory
+  file:
+    path: /etc/prometheus/exporter/ipmi
+    state: directory
+
+- name: generate configuration
+  template:
+    src: config.yml.j2
+    dest: /etc/prometheus/exporter/ipmi/config.yml
+  notify: reload prometheus-ipmi-exporter
+
+- name: generate systemd service unit
+  template:
+    src: service.j2
+    dest: /etc/systemd/system/prometheus-ipmi-exporter.service
+  notify: restart prometheus-ipmi-exporter
+
+- name: make sure prometheus-ipmi-exporter is enabled and started
+  systemd:
+    name: prometheus-ipmi-exporter.service
+    daemon_reload: yes
+    state: started
+    enabled: yes
+
+- name: register exporter
+  loop:
+  - name: local
+    path: /metrics
+  - name: remote
+    path: /ipmi
+  loop_control:
+    label: "{{ item.name }}"
+  copy:
+    content: |
+      method: http
+      http:
+        port: 9290
+        path: {{ item.path }}
+    dest: "/etc/prometheus/exporter/exporter/ipmi-{{ item.name }}.yml"
+  notify: reload prometheus-exporter-exporter
diff --git a/roles/monitoring/prometheus/exporter/ipmi/templates/config.yml.j2 b/roles/monitoring/prometheus/exporter/ipmi/templates/config.yml.j2
new file mode 100644
index 00000000..32d0b34a
--- /dev/null
+++ b/roles/monitoring/prometheus/exporter/ipmi/templates/config.yml.j2
@@ -0,0 +1,4 @@
+# {{ ansible_managed }}
+
+modules:
+  {{ prometheus_exporter_ipmi_modules | to_nice_yaml(indent=2) | indent(2) }}
diff --git a/roles/monitoring/prometheus/exporter/ipmi/templates/service.j2 b/roles/monitoring/prometheus/exporter/ipmi/templates/service.j2
new file mode 100644
index 00000000..465215e8
--- /dev/null
+++ b/roles/monitoring/prometheus/exporter/ipmi/templates/service.j2
@@ -0,0 +1,32 @@
+[Unit]
+Description=Prometheus ipmi exporter
+
+[Service]
+Restart=always
+User=prometheus-exporter
+ExecStart=/usr/bin/prometheus-ipmi-exporter --web.listen-address="127.0.0.1:9290" --config.file=/etc/prometheus/exporter/ipmi/config.yml --freeipmi.path="/usr/sbin"
+ExecReload=/bin/kill -HUP $MAINPID
+
+{# TODO: test which hardening options need to be removed for IPMI to work... #}
+# systemd hardening-options
+AmbientCapabilities=
+CapabilityBoundingSet=
+DeviceAllow=/dev/null rw
+DevicePolicy=strict
+LockPersonality=true
+MemoryDenyWriteExecute=true
+NoNewPrivileges=true
+PrivateDevices=true
+PrivateTmp=true
+ProtectControlGroups=true
+ProtectHome=true
+ProtectKernelModules=true
+ProtectKernelTunables=true
+ProtectSystem=strict
+RemoveIPC=true
+RestrictNamespaces=true
+RestrictRealtime=true
+SystemCallArchitectures=native
+
+[Install]
+WantedBy=multi-user.target
-- 
cgit v1.2.3


From 063bdb70a8e8353908ca9742e05be8fac65a61bf Mon Sep 17 00:00:00 2001
From: Christian Pointner <equinox@spreadspace.org>
Date: Sat, 25 Sep 2021 23:36:40 +0200
Subject: move away from exporter-exporter in favor for nginx

---
 chaos-at-home/ch-testvm-prometheus.yml             |  1 +
 inventory/host_vars/ch-testvm-prometheus.yml       |  3 ++
 .../prometheus/exporter/base/defaults/main.yml     |  2 +-
 .../prometheus/exporter/base/handlers/main.yml     |  6 ++--
 .../prometheus/exporter/base/tasks/main.yml        | 30 +++++---------------
 .../exporter/base/templates/nginx-vhost.j2         | 19 +++++++++++++
 .../prometheus/exporter/base/templates/service.j2  | 32 ----------------------
 .../prometheus/exporter/blackbox/handlers/main.yml |  7 ++---
 .../prometheus/exporter/blackbox/tasks/main.yml    | 11 ++++----
 .../prometheus/exporter/ipmi/handlers/main.yml     |  7 ++---
 .../prometheus/exporter/ipmi/tasks/main.yml        | 21 ++++++--------
 roles/monitoring/prometheus/exporter/meta/main.yml | 10 ++++---
 .../prometheus/exporter/mikrotik/handlers/main.yml |  7 ++---
 .../prometheus/exporter/mikrotik/tasks/main.yml    | 10 +++----
 .../prometheus/exporter/node/handlers/main.yml     |  7 ++---
 .../prometheus/exporter/node/tasks/main.yml        | 10 +++----
 .../prometheus/exporter/nut/handlers/main.yml      |  7 ++---
 .../prometheus/exporter/nut/tasks/main.yml         | 14 ++++++----
 .../server/templates/jobs/blackbox/https.j2        |  3 +-
 .../server/templates/jobs/blackbox/ping.j2         |  3 +-
 .../server/templates/jobs/blackbox/ssh.j2          |  3 +-
 .../prometheus/server/templates/jobs/generic.j2    |  5 +---
 .../prometheus/server/templates/jobs/node.j2       |  5 +---
 .../prometheus/server/templates/jobs/nut/ups.j2    |  5 +---
 24 files changed, 92 insertions(+), 136 deletions(-)
 create mode 100644 roles/monitoring/prometheus/exporter/base/templates/nginx-vhost.j2
 delete mode 100644 roles/monitoring/prometheus/exporter/base/templates/service.j2

(limited to 'roles/monitoring/prometheus/exporter/blackbox/tasks')

diff --git a/chaos-at-home/ch-testvm-prometheus.yml b/chaos-at-home/ch-testvm-prometheus.yml
index 3fd99d41..c0f33b8f 100644
--- a/chaos-at-home/ch-testvm-prometheus.yml
+++ b/chaos-at-home/ch-testvm-prometheus.yml
@@ -7,6 +7,7 @@
   - role: core/sshd/base
   - role: core/zsh
   - role: core/ntp
+  - role: nginx/base
   - role: apt-repo/spreadspace
   - role: monitoring/prometheus/exporter
   # - role: kubernetes/base
diff --git a/inventory/host_vars/ch-testvm-prometheus.yml b/inventory/host_vars/ch-testvm-prometheus.yml
index e539735f..939fa398 100644
--- a/inventory/host_vars/ch-testvm-prometheus.yml
+++ b/inventory/host_vars/ch-testvm-prometheus.yml
@@ -36,6 +36,9 @@ network:
 spreadspace_apt_repo_components:
   - prometheus
 
+prometheus_exporters_extra:
+  - ipmi
+
 
 containerd_storage:
   type: lvm
diff --git a/roles/monitoring/prometheus/exporter/base/defaults/main.yml b/roles/monitoring/prometheus/exporter/base/defaults/main.yml
index 963763a5..613943d8 100644
--- a/roles/monitoring/prometheus/exporter/base/defaults/main.yml
+++ b/roles/monitoring/prometheus/exporter/base/defaults/main.yml
@@ -1,2 +1,2 @@
 ---
-prometheus_exporter_listen: ":9999"
+prometheus_exporter_listen: "9999"
diff --git a/roles/monitoring/prometheus/exporter/base/handlers/main.yml b/roles/monitoring/prometheus/exporter/base/handlers/main.yml
index ebd760cf..d4e42ca0 100644
--- a/roles/monitoring/prometheus/exporter/base/handlers/main.yml
+++ b/roles/monitoring/prometheus/exporter/base/handlers/main.yml
@@ -1,5 +1,5 @@
 ---
-- name: restart prometheus-exporter-exporter
+- name: reload nginx
   service:
-    name: prometheus-exporter-exporter
-    state: restarted
+    name: nginx
+    state: reloaded
diff --git a/roles/monitoring/prometheus/exporter/base/tasks/main.yml b/roles/monitoring/prometheus/exporter/base/tasks/main.yml
index eeb2a23d..5f42867d 100644
--- a/roles/monitoring/prometheus/exporter/base/tasks/main.yml
+++ b/roles/monitoring/prometheus/exporter/base/tasks/main.yml
@@ -6,17 +6,6 @@
     - spreadspace_apt_repo_components is defined
     - "'prometheus' in spreadspace_apt_repo_components"
 
-  ## TODO: pin version
-- name: install apt packages
-  apt:
-    name: prom-exporter-exporter
-    state: present
-
-- name: create configuration directories
-  file:
-    path: /etc/prometheus/exporter/exporter
-    state: directory
-
 - name: add user for prometheus-exporter
   user:
     name: prometheus-exporter
@@ -27,15 +16,10 @@
 - name: create TLS certificate and key
   import_tasks: tls.yml
 
-- name: generate systemd service unit
-  template:
-    src: service.j2
-    dest: /etc/systemd/system/prometheus-exporter-exporter.service
-  notify: restart prometheus-exporter-exporter
-
-- name: make sure prometheus-exporter-exporter is enabled and started
-  systemd:
-    name: prometheus-exporter-exporter.service
-    daemon_reload: yes
-    state: started
-    enabled: yes
+- name: configure nginx vhost
+  import_role:
+    name: nginx/vhost
+  vars:
+    nginx_vhost:
+      name: prometheus-exporter
+      content: "{{ lookup('template', 'nginx-vhost.j2') }}"
diff --git a/roles/monitoring/prometheus/exporter/base/templates/nginx-vhost.j2 b/roles/monitoring/prometheus/exporter/base/templates/nginx-vhost.j2
new file mode 100644
index 00000000..70e65b29
--- /dev/null
+++ b/roles/monitoring/prometheus/exporter/base/templates/nginx-vhost.j2
@@ -0,0 +1,19 @@
+server {
+    listen {{ prometheus_exporter_listen }} ssl;
+    server_name _;
+
+    ssl_certificate /etc/ssl/prometheus/exporter/crt.pem;
+    ssl_certificate_key /etc/ssl/prometheus/exporter/key.pem;
+    ssl_client_certificate /etc/ssl/prometheus/ca-crt.pem;
+    ssl_verify_client on;
+
+    root /nonexistent;
+
+    location = / {
+        return 404 'please specify the exporter you want to reach!';
+    }
+
+    include snippets/proxy-nobuff.conf;
+
+    include /etc/prometheus/exporter/*.locations;
+}
diff --git a/roles/monitoring/prometheus/exporter/base/templates/service.j2 b/roles/monitoring/prometheus/exporter/base/templates/service.j2
deleted file mode 100644
index 3d44744a..00000000
--- a/roles/monitoring/prometheus/exporter/base/templates/service.j2
+++ /dev/null
@@ -1,32 +0,0 @@
-[Unit]
-Description=Prometheus exporter proxy
-
-[Service]
-Restart=always
-User=prometheus-exporter
-ExecStart=/usr/bin/prometheus-exporter-exporter -config.dirs=/etc/prometheus/exporter/exporter -config.file="" -web.listen-address="" -web.tls.listen-address="{{ prometheus_exporter_listen }}" -web.tls.cert="/etc/ssl/prometheus/exporter/crt.pem" -web.tls.key="/etc/ssl/prometheus/exporter/key.pem" --web.tls.ca="/etc/ssl/prometheus/ca-crt.pem" -web.tls.verify
-{# TODO: implement reloading once the exporter_exporter supports this #}
-
-# systemd hardening-options
-AmbientCapabilities=
-CapabilityBoundingSet=
-DeviceAllow=/dev/null rw
-DevicePolicy=strict
-LockPersonality=true
-MemoryDenyWriteExecute=true
-NoNewPrivileges=true
-PrivateDevices=true
-PrivateTmp=true
-PrivateUsers=true
-ProtectControlGroups=true
-ProtectHome=true
-ProtectKernelModules=true
-ProtectKernelTunables=true
-ProtectSystem=strict
-RemoveIPC=true
-RestrictNamespaces=true
-RestrictRealtime=true
-SystemCallArchitectures=native
-
-[Install]
-WantedBy=multi-user.target
diff --git a/roles/monitoring/prometheus/exporter/blackbox/handlers/main.yml b/roles/monitoring/prometheus/exporter/blackbox/handlers/main.yml
index 99a416e2..12250769 100644
--- a/roles/monitoring/prometheus/exporter/blackbox/handlers/main.yml
+++ b/roles/monitoring/prometheus/exporter/blackbox/handlers/main.yml
@@ -9,8 +9,7 @@
     name: prometheus-blackbox-exporter
     state: reloaded
 
-- name: reload prometheus-exporter-exporter
+- name: reload nginx
   service:
-    name: prometheus-exporter-exporter
-    ## TODO: implement reload once exporter_exporter supports this...
-    state: restarted
+    name: nginx
+    state: reloaded
diff --git a/roles/monitoring/prometheus/exporter/blackbox/tasks/main.yml b/roles/monitoring/prometheus/exporter/blackbox/tasks/main.yml
index 782c3561..f9793df6 100644
--- a/roles/monitoring/prometheus/exporter/blackbox/tasks/main.yml
+++ b/roles/monitoring/prometheus/exporter/blackbox/tasks/main.yml
@@ -32,9 +32,8 @@
 - name: register exporter
   copy:
     content: |
-      method: http
-      http:
-        port: 9115
-        path: /probe
-    dest: /etc/prometheus/exporter/exporter/blackbox.yml
-  notify: reload prometheus-exporter-exporter
+      location = /blackbox {
+          proxy_pass http://127.0.0.1:9115/probe;
+      }
+    dest: /etc/prometheus/exporter/blackbox.locations
+  notify: reload nginx
diff --git a/roles/monitoring/prometheus/exporter/ipmi/handlers/main.yml b/roles/monitoring/prometheus/exporter/ipmi/handlers/main.yml
index 40a945ae..a8eb55b3 100644
--- a/roles/monitoring/prometheus/exporter/ipmi/handlers/main.yml
+++ b/roles/monitoring/prometheus/exporter/ipmi/handlers/main.yml
@@ -9,8 +9,7 @@
     name: prometheus-ipmi-exporter
     state: reloaded
 
-- name: reload prometheus-exporter-exporter
+- name: reload nginx
   service:
-    name: prometheus-exporter-exporter
-    ## TODO: implement reload once exporter_exporter supports this...
-    state: restarted
+    name: nginx
+    state: reloaded
diff --git a/roles/monitoring/prometheus/exporter/ipmi/tasks/main.yml b/roles/monitoring/prometheus/exporter/ipmi/tasks/main.yml
index 9e63f692..91318f16 100644
--- a/roles/monitoring/prometheus/exporter/ipmi/tasks/main.yml
+++ b/roles/monitoring/prometheus/exporter/ipmi/tasks/main.yml
@@ -30,18 +30,13 @@
     enabled: yes
 
 - name: register exporter
-  loop:
-  - name: local
-    path: /metrics
-  - name: remote
-    path: /ipmi
-  loop_control:
-    label: "{{ item.name }}"
   copy:
     content: |
-      method: http
-      http:
-        port: 9290
-        path: {{ item.path }}
-    dest: "/etc/prometheus/exporter/exporter/ipmi-{{ item.name }}.yml"
-  notify: reload prometheus-exporter-exporter
+      location = /ipmi {
+          proxy_pass http://127.0.0.1:9290/metrics;
+      }
+      location = /ipmi/remote {
+          proxy_pass http://127.0.0.1:9290/ipmi;
+      }
+    dest: /etc/prometheus/exporter/ipmi.locations
+  notify: reload nginx
diff --git a/roles/monitoring/prometheus/exporter/meta/main.yml b/roles/monitoring/prometheus/exporter/meta/main.yml
index 22131422..68fce6cb 100644
--- a/roles/monitoring/prometheus/exporter/meta/main.yml
+++ b/roles/monitoring/prometheus/exporter/meta/main.yml
@@ -1,11 +1,13 @@
 ---
 dependencies:
   - role: monitoring/prometheus/exporter/base
-  - role: monitoring/prometheus/exporter/node
-    when: "'node' in (prometheus_exporters_default | union(prometheus_exporters_extra))"
   - role: monitoring/prometheus/exporter/blackbox
     when: "'blackbox' in (prometheus_exporters_default | union(prometheus_exporters_extra))"
-  - role: monitoring/prometheus/exporter/nut
-    when: "'nut' in (prometheus_exporters_default | union(prometheus_exporters_extra))"
+  - role: monitoring/prometheus/exporter/ipmi
+    when: "'ipmi' in (prometheus_exporters_default | union(prometheus_exporters_extra))"
   - role: monitoring/prometheus/exporter/mikrotik
     when: "'mikrotik' in (prometheus_exporters_default | union(prometheus_exporters_extra))"
+  - role: monitoring/prometheus/exporter/node
+    when: "'node' in (prometheus_exporters_default | union(prometheus_exporters_extra))"
+  - role: monitoring/prometheus/exporter/nut
+    when: "'nut' in (prometheus_exporters_default | union(prometheus_exporters_extra))"
diff --git a/roles/monitoring/prometheus/exporter/mikrotik/handlers/main.yml b/roles/monitoring/prometheus/exporter/mikrotik/handlers/main.yml
index cb85d0d9..c5844220 100644
--- a/roles/monitoring/prometheus/exporter/mikrotik/handlers/main.yml
+++ b/roles/monitoring/prometheus/exporter/mikrotik/handlers/main.yml
@@ -4,8 +4,7 @@
     name: prometheus-mikrotik-exporter
     state: restarted
 
-- name: reload prometheus-exporter-exporter
+- name: reload nginx
   service:
-    name: prometheus-exporter-exporter
-    ## TODO: implement reload once exporter_exporter supports this...
-    state: restarted
+    name: nginx
+    state: reloaded
diff --git a/roles/monitoring/prometheus/exporter/mikrotik/tasks/main.yml b/roles/monitoring/prometheus/exporter/mikrotik/tasks/main.yml
index 07219c68..72c78e4a 100644
--- a/roles/monitoring/prometheus/exporter/mikrotik/tasks/main.yml
+++ b/roles/monitoring/prometheus/exporter/mikrotik/tasks/main.yml
@@ -35,8 +35,8 @@
 - name: register exporter
   copy:
     content: |
-      method: http
-      http:
-        port: 9436
-    dest: /etc/prometheus/exporter/exporter/mikrotik.yml
-  notify: reload prometheus-exporter-exporter
+      location = /mikrotik {
+          proxy_pass http://127.0.0.1:9436/metrics;
+      }
+    dest: /etc/prometheus/exporter/mikrotik.locations
+  notify: reload nginx
diff --git a/roles/monitoring/prometheus/exporter/node/handlers/main.yml b/roles/monitoring/prometheus/exporter/node/handlers/main.yml
index 3e1b2000..56056ea6 100644
--- a/roles/monitoring/prometheus/exporter/node/handlers/main.yml
+++ b/roles/monitoring/prometheus/exporter/node/handlers/main.yml
@@ -4,8 +4,7 @@
     name: prometheus-node-exporter
     state: restarted
 
-- name: reload prometheus-exporter-exporter
+- name: reload nginx
   service:
-    name: prometheus-exporter-exporter
-    ## TODO: implement reload once exporter_exporter supports this...
-    state: restarted
+    name: nginx
+    state: reloaded
diff --git a/roles/monitoring/prometheus/exporter/node/tasks/main.yml b/roles/monitoring/prometheus/exporter/node/tasks/main.yml
index 56903a33..2811c759 100644
--- a/roles/monitoring/prometheus/exporter/node/tasks/main.yml
+++ b/roles/monitoring/prometheus/exporter/node/tasks/main.yml
@@ -28,11 +28,11 @@
 - name: register exporter
   copy:
     content: |
-      method: http
-      http:
-        port: 9100
-    dest: /etc/prometheus/exporter/exporter/node.yml
-  notify: reload prometheus-exporter-exporter
+      location = /node {
+          proxy_pass http://127.0.0.1:9100/metrics;
+      }
+    dest: /etc/prometheus/exporter/node.locations
+  notify: reload nginx
 
 - name: create directory for textfile collector scripts
   file:
diff --git a/roles/monitoring/prometheus/exporter/nut/handlers/main.yml b/roles/monitoring/prometheus/exporter/nut/handlers/main.yml
index 6e10f43b..edd87ed5 100644
--- a/roles/monitoring/prometheus/exporter/nut/handlers/main.yml
+++ b/roles/monitoring/prometheus/exporter/nut/handlers/main.yml
@@ -4,8 +4,7 @@
     name: prometheus-nut-exporter
     state: restarted
 
-- name: reload prometheus-exporter-exporter
+- name: reload ngnix
   service:
-    name: prometheus-exporter-exporter
-    ## TODO: implement reload once exporter_exporter supports this...
-    state: restarted
+    name: nginx
+    state: reloaded
diff --git a/roles/monitoring/prometheus/exporter/nut/tasks/main.yml b/roles/monitoring/prometheus/exporter/nut/tasks/main.yml
index 8245feae..f602472d 100644
--- a/roles/monitoring/prometheus/exporter/nut/tasks/main.yml
+++ b/roles/monitoring/prometheus/exporter/nut/tasks/main.yml
@@ -21,9 +21,11 @@
 - name: register exporter
   copy:
     content: |
-      method: http
-      http:
-        port: 9199
-        path: /ups_metrics
-    dest: /etc/prometheus/exporter/exporter/nut.yml
-  notify: reload prometheus-exporter-exporter
+      location = /nut {
+          proxy_pass http://127.0.0.1:9199/metrics;
+      }
+      location = /nut/ups {
+          proxy_pass http://127.0.0.1:9199/ups_metrics;
+      }
+    dest: /etc/prometheus/exporter/nut.locations
+  notify: reload nginx
diff --git a/roles/monitoring/prometheus/server/templates/jobs/blackbox/https.j2 b/roles/monitoring/prometheus/server/templates/jobs/blackbox/https.j2
index 98a64121..86ff88dd 100644
--- a/roles/monitoring/prometheus/server/templates/jobs/blackbox/https.j2
+++ b/roles/monitoring/prometheus/server/templates/jobs/blackbox/https.j2
@@ -1,8 +1,7 @@
   - job_name: '{{ job }}'
-    metrics_path: /proxy
+    metrics_path: /blackbox
     params:
       module:
-      - blackbox
       - http_tls_2xx
     scheme: https
     tls_config:
diff --git a/roles/monitoring/prometheus/server/templates/jobs/blackbox/ping.j2 b/roles/monitoring/prometheus/server/templates/jobs/blackbox/ping.j2
index 736ffec1..2d3889d2 100644
--- a/roles/monitoring/prometheus/server/templates/jobs/blackbox/ping.j2
+++ b/roles/monitoring/prometheus/server/templates/jobs/blackbox/ping.j2
@@ -1,8 +1,7 @@
   - job_name: '{{ job }}'
-    metrics_path: /proxy
+    metrics_path: /blackbox
     params:
       module:
-      - blackbox
       - icmp
     scheme: https
     tls_config:
diff --git a/roles/monitoring/prometheus/server/templates/jobs/blackbox/ssh.j2 b/roles/monitoring/prometheus/server/templates/jobs/blackbox/ssh.j2
index 166f37ad..97565673 100644
--- a/roles/monitoring/prometheus/server/templates/jobs/blackbox/ssh.j2
+++ b/roles/monitoring/prometheus/server/templates/jobs/blackbox/ssh.j2
@@ -1,8 +1,7 @@
   - job_name: '{{ job }}'
-    metrics_path: /proxy
+    metrics_path: /blackbox
     params:
       module:
-      - blackbox
       - ssh_banner
     scheme: https
     tls_config:
diff --git a/roles/monitoring/prometheus/server/templates/jobs/generic.j2 b/roles/monitoring/prometheus/server/templates/jobs/generic.j2
index b155c5f7..65a95007 100644
--- a/roles/monitoring/prometheus/server/templates/jobs/generic.j2
+++ b/roles/monitoring/prometheus/server/templates/jobs/generic.j2
@@ -1,8 +1,5 @@
   - job_name: '{{ job }}'
-    metrics_path: /proxy
-    params:
-      module:
-      - {{ job }}
+    metrics_path: /{{ job }}
     scheme: https
     tls_config:
       ca_file: /etc/ssl/prometheus/ca-crt.pem
diff --git a/roles/monitoring/prometheus/server/templates/jobs/node.j2 b/roles/monitoring/prometheus/server/templates/jobs/node.j2
index ba9eab31..1b14e1f6 100644
--- a/roles/monitoring/prometheus/server/templates/jobs/node.j2
+++ b/roles/monitoring/prometheus/server/templates/jobs/node.j2
@@ -1,8 +1,5 @@
   - job_name: '{{ job }}'
-    metrics_path: /proxy
-    params:
-      module:
-      - {{ job }}
+    metrics_path: /{{ job }}
     scheme: https
     tls_config:
       ca_file: /etc/ssl/prometheus/ca-crt.pem
diff --git a/roles/monitoring/prometheus/server/templates/jobs/nut/ups.j2 b/roles/monitoring/prometheus/server/templates/jobs/nut/ups.j2
index 3a2c5c62..0cf4ae4e 100644
--- a/roles/monitoring/prometheus/server/templates/jobs/nut/ups.j2
+++ b/roles/monitoring/prometheus/server/templates/jobs/nut/ups.j2
@@ -1,8 +1,5 @@
   - job_name: '{{ job }}'
-    metrics_path: /proxy
-    params:
-      module:
-      - nut
+    metrics_path: /nut/ups
     scheme: https
     tls_config:
       ca_file: /etc/ssl/prometheus/ca-crt.pem
-- 
cgit v1.2.3


From cc89d6d4211aa5aec8e5bef8c854d4929c337887 Mon Sep 17 00:00:00 2001
From: Christian Pointner <equinox@spreadspace.org>
Date: Sun, 26 Sep 2021 03:32:47 +0200
Subject: improved promethues multitarget support

---
 inventory/group_vars/chaos-at-home-ups/vars.yml    |  6 +-
 inventory/group_vars/ele-ups/vars.yml              | 10 +--
 .../group_vars/promzone-chaos-at-home/vars.yml     |  6 +-
 .../group_vars/promzone-elevate-festival/vars.yml  |  1 +
 inventory/host_vars/ch-mon.yml                     | 39 ++++++------
 .../prometheus/exporter/blackbox/tasks/main.yml    |  3 +
 .../prometheus/server/defaults/main/main.yml       |  4 +-
 .../server/defaults/main/rules_blackbox.yml        | 46 +-------------
 .../server/defaults/main/rules_blackbox__https.yml | 20 ------
 .../server/defaults/main/rules_blackbox__ping.yml  | 11 ----
 .../server/defaults/main/rules_blackbox__probe.yml | 74 ++++++++++++++++++++++
 .../server/defaults/main/rules_blackbox__ssh.yml   |  3 -
 .../prometheus/server/filter_plugins/prometheus.py | 10 +--
 roles/monitoring/prometheus/server/tasks/main.yml  |  2 +-
 .../server/templates/jobs/blackbox/https.j2        | 13 ----
 .../server/templates/jobs/blackbox/ping.j2         | 13 ----
 .../server/templates/jobs/blackbox/ssh.j2          | 13 ----
 .../prometheus/server/templates/jobs/nut/ups.j2    | 10 ---
 .../server/templates/targets/blackbox/https.yml.j2 |  4 --
 .../server/templates/targets/blackbox/ping.yml.j2  |  4 --
 .../server/templates/targets/blackbox/probe.yml.j2 |  5 ++
 .../server/templates/targets/blackbox/ssh.yml.j2   |  4 --
 .../server/templates/targets/nut/ups.yml.j2        |  2 +-
 23 files changed, 123 insertions(+), 180 deletions(-)
 delete mode 100644 roles/monitoring/prometheus/server/defaults/main/rules_blackbox__https.yml
 delete mode 100644 roles/monitoring/prometheus/server/defaults/main/rules_blackbox__ping.yml
 create mode 100644 roles/monitoring/prometheus/server/defaults/main/rules_blackbox__probe.yml
 delete mode 100644 roles/monitoring/prometheus/server/defaults/main/rules_blackbox__ssh.yml
 delete mode 100644 roles/monitoring/prometheus/server/templates/jobs/blackbox/https.j2
 delete mode 100644 roles/monitoring/prometheus/server/templates/jobs/blackbox/ping.j2
 delete mode 100644 roles/monitoring/prometheus/server/templates/jobs/blackbox/ssh.j2
 delete mode 100644 roles/monitoring/prometheus/server/templates/jobs/nut/ups.j2
 delete mode 100644 roles/monitoring/prometheus/server/templates/targets/blackbox/https.yml.j2
 delete mode 100644 roles/monitoring/prometheus/server/templates/targets/blackbox/ping.yml.j2
 create mode 100644 roles/monitoring/prometheus/server/templates/targets/blackbox/probe.yml.j2
 delete mode 100644 roles/monitoring/prometheus/server/templates/targets/blackbox/ssh.yml.j2

(limited to 'roles/monitoring/prometheus/exporter/blackbox/tasks')

diff --git a/inventory/group_vars/chaos-at-home-ups/vars.yml b/inventory/group_vars/chaos-at-home-ups/vars.yml
index f8c1bdf1..7b60e893 100644
--- a/inventory/group_vars/chaos-at-home-ups/vars.yml
+++ b/inventory/group_vars/chaos-at-home-ups/vars.yml
@@ -11,8 +11,8 @@ prometheus_scrape_endpoint: "{{ network_mgmt_zone.prefix | ipaddr(network_mgmt_z
 prometheus_exporters_default:
   - openwrt
 
-prometheus_job_nut__ups:
-  - exporter_hostname: ch-mon
-    instance: "ups-{{ ups_name }}"
+prometheus_job_multitarget_nut__ups:
+  ch-mon:
+  - instance: "ups-{{ ups_name }}"
     ups: "{{ ups_name }}"
     server: "{{ network_mgmt_zone.prefix | ipaddr(network_mgmt_zone.offsets[inventory_hostname]) | ipaddr('address') }}"
diff --git a/inventory/group_vars/ele-ups/vars.yml b/inventory/group_vars/ele-ups/vars.yml
index 1c4613a3..28a5eaff 100644
--- a/inventory/group_vars/ele-ups/vars.yml
+++ b/inventory/group_vars/ele-ups/vars.yml
@@ -14,8 +14,8 @@ prometheus_scrape_endpoint: "{{ network_mgmt_zone.prefix | ipaddr(network_mgmt_z
 prometheus_exporters_default:
   - openwrt
 
-prometheus_job_nut__ups:
-  exporter_hostname: ele-mon
-  instance: "ups-{{ ups_name }}"
-  ups: "{{ ups_name }}"
-  server: "{{ network_mgmt_zone.prefix | ipaddr(network_mgmt_zone.offsets[inventory_hostname]) | ipaddr('address') }}"
+prometheus_job_multitarget_nut__ups:
+  ele-mon:
+  - instance: "ups-{{ ups_name }}"
+    ups: "{{ ups_name }}"
+    server: "{{ network_mgmt_zone.prefix | ipaddr(network_mgmt_zone.offsets[inventory_hostname]) | ipaddr('address') }}"
diff --git a/inventory/group_vars/promzone-chaos-at-home/vars.yml b/inventory/group_vars/promzone-chaos-at-home/vars.yml
index 84ed1263..529bf3e7 100644
--- a/inventory/group_vars/promzone-chaos-at-home/vars.yml
+++ b/inventory/group_vars/promzone-chaos-at-home/vars.yml
@@ -9,10 +9,10 @@ prometheus_server: ch-mon
 prometheus_server_jobs:
   - node
   - openwrt
+  - nut
   - nut/ups
-  - blackbox/ping
-  - blackbox/https
-  - blackbox/ssh
+  - blackbox
+  - blackbox/probe
 
 prometheus_zone_name: chaos@home
 prometheus_zone_targets: "{{ groups['promzone-chaos-at-home'] }}"
diff --git a/inventory/group_vars/promzone-elevate-festival/vars.yml b/inventory/group_vars/promzone-elevate-festival/vars.yml
index 43115dc4..b3321614 100644
--- a/inventory/group_vars/promzone-elevate-festival/vars.yml
+++ b/inventory/group_vars/promzone-elevate-festival/vars.yml
@@ -9,6 +9,7 @@ prometheus_server: ele-mon
 prometheus_server_jobs:
   - node
   - openwrt
+  - nut
   - nut/ups
 
 prometheus_zone_name: Elevate Festival
diff --git a/inventory/host_vars/ch-mon.yml b/inventory/host_vars/ch-mon.yml
index d1a710b9..b2402d0c 100644
--- a/inventory/host_vars/ch-mon.yml
+++ b/inventory/host_vars/ch-mon.yml
@@ -76,26 +76,25 @@ prometheus_exporter_blackbox_modules_extra:
   icmp:
     prober: icmp
 
-prometheus_job_blackbox__ping:
-  - exporter_hostname: ch-mon
-    instance: "ping-magentagw"
-    address: 62.99.185.129
-  - exporter_hostname: ch-mon
-    instance: "ping-quad9"
-    address: 9.9.9.9
-
-prometheus_job_blackbox__https:
-  - exporter_hostname: ch-mon
-    instance: "https-pan.chaos-at-home.org"
-    address: "https://pan.chaos-at-home.org"
-  - exporter_hostname: ch-mon
-    instance: "https-mimas.chaos-at-home.org"
-    address: "https://mimas.chaos-at-home.org"
-
-prometheus_job_blackbox__ssh:
-  - exporter_hostname: ch-mon
-    instance: "ssh-{{ inventory_hostname }}"
-    address: "{{ network_zones.svc.prefix | ipaddr(network_zones.svc.offsets[inventory_hostname]) | ipaddr('address') }}:{{ ansible_port | default(22) }}"
+prometheus_job_multitarget_blackbox__probe:
+  ch-mon:
+  - instance: "ping-magentagw"
+    target: 62.99.185.129
+    module: icmp
+  - instance: "ping-quad9"
+    target: 9.9.9.9
+    module: icmp
+
+  - instance: "https-pan.chaos-at-home.org"
+    target: "https://pan.chaos-at-home.org"
+    module: http_tls_2xx
+  - instance: "https-mimas.chaos-at-home.org"
+    target: "https://mimas.chaos-at-home.org"
+    module: http_tls_2xx
+
+  - instance: "ssh-{{ inventory_hostname }}"
+    target: "{{ network_zones.svc.prefix | ipaddr(network_zones.svc.offsets[inventory_hostname]) | ipaddr('address') }}:{{ ansible_port | default(22) }}"
+    module: ssh_banner
 
 
 promethues_alertmanager_smtp:
diff --git a/roles/monitoring/prometheus/exporter/blackbox/tasks/main.yml b/roles/monitoring/prometheus/exporter/blackbox/tasks/main.yml
index f9793df6..c4cabfce 100644
--- a/roles/monitoring/prometheus/exporter/blackbox/tasks/main.yml
+++ b/roles/monitoring/prometheus/exporter/blackbox/tasks/main.yml
@@ -33,6 +33,9 @@
   copy:
     content: |
       location = /blackbox {
+          proxy_pass http://127.0.0.1:9115/metrics;
+      }
+      location = /blackbox/probe {
           proxy_pass http://127.0.0.1:9115/probe;
       }
     dest: /etc/prometheus/exporter/blackbox.locations
diff --git a/roles/monitoring/prometheus/server/defaults/main/main.yml b/roles/monitoring/prometheus/server/defaults/main/main.yml
index 09cd150c..7781fd69 100644
--- a/roles/monitoring/prometheus/server/defaults/main/main.yml
+++ b/roles/monitoring/prometheus/server/defaults/main/main.yml
@@ -17,9 +17,7 @@ prometheus_server_rules:
   nut: "{{ prometheus_server_rules_nut + prometheus_server_rules_nut_extra }}"
   nut/ups: "{{ prometheus_server_rules_nut__ups + prometheus_server_rules_nut__ups_extra }}"
   blackbox: "{{ prometheus_server_rules_blackbox + prometheus_server_rules_blackbox_extra }}"
-  blackbox/ping: "{{ prometheus_server_rules_blackbox__ping + prometheus_server_rules_blackbox__ping_extra }}"
-  blackbox/https: "{{ prometheus_server_rules_blackbox__https + prometheus_server_rules_blackbox__https_extra }}"
-  blackbox/ssh: "{{ prometheus_server_rules_blackbox__ssh + prometheus_server_rules_blackbox__ssh_extra }}"
+  blackbox/probe: "{{ prometheus_server_rules_blackbox__probe + prometheus_server_rules_blackbox__probe_extra }}"
 
 # prometheus_server_alertmanager:
 #   url: "127.0.0.1:9093"
diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_blackbox.yml b/roles/monitoring/prometheus/server/defaults/main/rules_blackbox.yml
index d5c1fd42..99f2e83c 100644
--- a/roles/monitoring/prometheus/server/defaults/main/rules_blackbox.yml
+++ b/roles/monitoring/prometheus/server/defaults/main/rules_blackbox.yml
@@ -1,47 +1,3 @@
 ---
 prometheus_server_rules_blackbox_extra: []
-prometheus_server_rules_blackbox:
-  - alert: BlackboxProbeFailed
-    expr: probe_success == 0
-    for: 0m
-    labels:
-      severity: critical
-    annotations:
-      summary: Blackbox probe failed (instance {{ '{{' }} $labels.instance {{ '}}' }})
-      description: "Probe failed\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
-
-  - alert: BlackboxSlowProbe
-    expr: avg_over_time(probe_duration_seconds[1m]) > 1
-    for: 1m
-    labels:
-      severity: warning
-    annotations:
-      summary: Blackbox slow probe (instance {{ '{{' }} $labels.instance {{ '}}' }})
-      description: "Blackbox probe took more than 1s to complete\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
-
-  - alert: BlackboxSslCertificateWillExpireSoon
-    expr: probe_ssl_earliest_cert_expiry - time() < 86400 * 30
-    for: 0m
-    labels:
-      severity: warning
-    annotations:
-      summary: Blackbox SSL certificate will expire soon (instance {{ '{{' }} $labels.instance {{ '}}' }})
-      description: "SSL certificate expires in 30 days\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
-
-  - alert: BlackboxSslCertificateWillExpireSoon
-    expr: probe_ssl_earliest_cert_expiry - time() < 86400 * 3
-    for: 0m
-    labels:
-      severity: critical
-    annotations:
-      summary: Blackbox SSL certificate will expire soon (instance {{ '{{' }} $labels.instance {{ '}}' }})
-      description: "SSL certificate expires in 3 days\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
-
-  - alert: BlackboxSslCertificateExpired
-    expr: probe_ssl_earliest_cert_expiry - time() <= 0
-    for: 0m
-    labels:
-      severity: critical
-    annotations:
-      summary: Blackbox SSL certificate expired (instance {{ '{{' }} $labels.instance {{ '}}' }})
-      description: "SSL certificate has expired already\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+prometheus_server_rules_blackbox: []
diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_blackbox__https.yml b/roles/monitoring/prometheus/server/defaults/main/rules_blackbox__https.yml
deleted file mode 100644
index 140e3b4f..00000000
--- a/roles/monitoring/prometheus/server/defaults/main/rules_blackbox__https.yml
+++ /dev/null
@@ -1,20 +0,0 @@
----
-prometheus_server_rules_blackbox__https_extra: []
-prometheus_server_rules_blackbox__https:
-  - alert: BlackboxProbeHttpFailure
-    expr: probe_http_status_code <= 199 OR probe_http_status_code >= 400
-    for: 0m
-    labels:
-      severity: critical
-    annotations:
-      summary: Blackbox probe HTTP failure (instance {{ '{{' }} $labels.instance {{ '}}' }})
-      description: "HTTP status code is not 200-399\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
-
-  - alert: BlackboxProbeSlowHttp
-    expr: avg_over_time(probe_http_duration_seconds[1m]) > 1
-    for: 1m
-    labels:
-      severity: warning
-    annotations:
-      summary: Blackbox probe slow HTTP (instance {{ '{{' }} $labels.instance {{ '}}' }})
-      description: "HTTP request took more than 1s\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_blackbox__ping.yml b/roles/monitoring/prometheus/server/defaults/main/rules_blackbox__ping.yml
deleted file mode 100644
index cc87b6b1..00000000
--- a/roles/monitoring/prometheus/server/defaults/main/rules_blackbox__ping.yml
+++ /dev/null
@@ -1,11 +0,0 @@
----
-prometheus_server_rules_blackbox__ping_extra: []
-prometheus_server_rules_blackbox__ping:
-  - alert: BlackboxProbeSlowPing
-    expr: avg_over_time(probe_icmp_duration_seconds[1m]) > 1
-    for: 1m
-    labels:
-      severity: warning
-    annotations:
-      summary: Blackbox probe slow ping (instance {{ '{{' }} $labels.instance {{ '}}' }})
-      description: "Blackbox ping took more than 1s\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_blackbox__probe.yml b/roles/monitoring/prometheus/server/defaults/main/rules_blackbox__probe.yml
new file mode 100644
index 00000000..9f9d2292
--- /dev/null
+++ b/roles/monitoring/prometheus/server/defaults/main/rules_blackbox__probe.yml
@@ -0,0 +1,74 @@
+---
+prometheus_server_rules_blackbox__probe_extra: []
+prometheus_server_rules_blackbox__probe:
+  - alert: BlackboxProbeFailed
+    expr: probe_success == 0
+    for: 0m
+    labels:
+      severity: critical
+    annotations:
+      summary: Blackbox probe failed (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "Probe failed\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: BlackboxSlowProbe
+    expr: avg_over_time(probe_duration_seconds[1m]) > 1
+    for: 1m
+    labels:
+      severity: warning
+    annotations:
+      summary: Blackbox slow probe (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "Blackbox probe took more than 1s to complete\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: BlackboxSslCertificateWillExpireSoon
+    expr: probe_ssl_earliest_cert_expiry - time() < 86400 * 30
+    for: 0m
+    labels:
+      severity: warning
+    annotations:
+      summary: Blackbox SSL certificate will expire soon (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "SSL certificate expires in 30 days\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: BlackboxSslCertificateWillExpireSoon
+    expr: probe_ssl_earliest_cert_expiry - time() < 86400 * 3
+    for: 0m
+    labels:
+      severity: critical
+    annotations:
+      summary: Blackbox SSL certificate will expire soon (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "SSL certificate expires in 3 days\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: BlackboxSslCertificateExpired
+    expr: probe_ssl_earliest_cert_expiry - time() <= 0
+    for: 0m
+    labels:
+      severity: critical
+    annotations:
+      summary: Blackbox SSL certificate expired (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "SSL certificate has expired already\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: BlackboxProbeHttpFailure
+    expr: probe_http_status_code <= 199 OR probe_http_status_code >= 400
+    for: 0m
+    labels:
+      severity: critical
+    annotations:
+      summary: Blackbox probe HTTP failure (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "HTTP status code is not 200-399\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: BlackboxProbeSlowHttp
+    expr: avg_over_time(probe_http_duration_seconds[1m]) > 1
+    for: 1m
+    labels:
+      severity: warning
+    annotations:
+      summary: Blackbox probe slow HTTP (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "HTTP request took more than 1s\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: BlackboxProbeSlowPing
+    expr: avg_over_time(probe_icmp_duration_seconds[1m]) > 1
+    for: 1m
+    labels:
+      severity: warning
+    annotations:
+      summary: Blackbox probe slow ping (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "Blackbox ping took more than 1s\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_blackbox__ssh.yml b/roles/monitoring/prometheus/server/defaults/main/rules_blackbox__ssh.yml
deleted file mode 100644
index 8e717c41..00000000
--- a/roles/monitoring/prometheus/server/defaults/main/rules_blackbox__ssh.yml
+++ /dev/null
@@ -1,3 +0,0 @@
----
-prometheus_server_rules_blackbox__ssh_extra: []
-prometheus_server_rules_blackbox__ssh: []
diff --git a/roles/monitoring/prometheus/server/filter_plugins/prometheus.py b/roles/monitoring/prometheus/server/filter_plugins/prometheus.py
index 1443e837..d91ef619 100644
--- a/roles/monitoring/prometheus/server/filter_plugins/prometheus.py
+++ b/roles/monitoring/prometheus/server/filter_plugins/prometheus.py
@@ -11,10 +11,12 @@ def prometheus_job_targets(hostvars, jobs, targets):
         result = []
         for job in jobs:
             for target in targets:
-                special_config_varname = 'prometheus_job_' + job.replace('-', '_').replace('/', '__')
-                if special_config_varname in hostvars[target]:
-                    for config in hostvars[target][special_config_varname]:
-                        result.append({'job': job, 'instance': config['instance'], 'config': config, 'enabled': True})
+                multitarget_config_varname = 'prometheus_job_multitarget_' + job.replace('-', '_').replace('/', '__')
+                if multitarget_config_varname in hostvars[target]:
+                    for exporter_hostname, configs in hostvars[target][multitarget_config_varname].items():
+                        for config in configs:
+                            result.append({'job': job, 'instance': config['instance'], 'enabled': True,
+                                           'exporter_hostname': exporter_hostname, 'config': config})
 
                 else:
                     enabled = job in hostvars[target]['prometheus_exporters_default'] or job in hostvars[target]['prometheus_exporters_extra']
diff --git a/roles/monitoring/prometheus/server/tasks/main.yml b/roles/monitoring/prometheus/server/tasks/main.yml
index c0928cc3..16167c9c 100644
--- a/roles/monitoring/prometheus/server/tasks/main.yml
+++ b/roles/monitoring/prometheus/server/tasks/main.yml
@@ -83,7 +83,7 @@
     state: directory
 
 - name: generate rules files for all jobs
-  loop: "{{ prometheus_server_jobs | union(['prometheus']) | union(prometheus_server_jobs | select('match', '.*/.*') | map('dirname') | unique) }}"
+  loop: "{{ prometheus_server_jobs | union(['prometheus']) }}"
   template:
     src: rules.yml.j2
     dest: "/etc/prometheus/rules/{{ item }}.yml"
diff --git a/roles/monitoring/prometheus/server/templates/jobs/blackbox/https.j2 b/roles/monitoring/prometheus/server/templates/jobs/blackbox/https.j2
deleted file mode 100644
index 86ff88dd..00000000
--- a/roles/monitoring/prometheus/server/templates/jobs/blackbox/https.j2
+++ /dev/null
@@ -1,13 +0,0 @@
-  - job_name: '{{ job }}'
-    metrics_path: /blackbox
-    params:
-      module:
-      - http_tls_2xx
-    scheme: https
-    tls_config:
-      ca_file: /etc/ssl/prometheus/ca-crt.pem
-      cert_file: /etc/ssl/prometheus/server/scrape-crt.pem
-      key_file: /etc/ssl/prometheus/server/scrape-key.pem
-    file_sd_configs:
-    - files:
-      - "/etc/prometheus/targets/{{ job }}/*.yml"
diff --git a/roles/monitoring/prometheus/server/templates/jobs/blackbox/ping.j2 b/roles/monitoring/prometheus/server/templates/jobs/blackbox/ping.j2
deleted file mode 100644
index 2d3889d2..00000000
--- a/roles/monitoring/prometheus/server/templates/jobs/blackbox/ping.j2
+++ /dev/null
@@ -1,13 +0,0 @@
-  - job_name: '{{ job }}'
-    metrics_path: /blackbox
-    params:
-      module:
-      - icmp
-    scheme: https
-    tls_config:
-      ca_file: /etc/ssl/prometheus/ca-crt.pem
-      cert_file: /etc/ssl/prometheus/server/scrape-crt.pem
-      key_file: /etc/ssl/prometheus/server/scrape-key.pem
-    file_sd_configs:
-    - files:
-      - "/etc/prometheus/targets/{{ job }}/*.yml"
diff --git a/roles/monitoring/prometheus/server/templates/jobs/blackbox/ssh.j2 b/roles/monitoring/prometheus/server/templates/jobs/blackbox/ssh.j2
deleted file mode 100644
index 97565673..00000000
--- a/roles/monitoring/prometheus/server/templates/jobs/blackbox/ssh.j2
+++ /dev/null
@@ -1,13 +0,0 @@
-  - job_name: '{{ job }}'
-    metrics_path: /blackbox
-    params:
-      module:
-      - ssh_banner
-    scheme: https
-    tls_config:
-      ca_file: /etc/ssl/prometheus/ca-crt.pem
-      cert_file: /etc/ssl/prometheus/server/scrape-crt.pem
-      key_file: /etc/ssl/prometheus/server/scrape-key.pem
-    file_sd_configs:
-    - files:
-      - "/etc/prometheus/targets/{{ job }}/*.yml"
diff --git a/roles/monitoring/prometheus/server/templates/jobs/nut/ups.j2 b/roles/monitoring/prometheus/server/templates/jobs/nut/ups.j2
deleted file mode 100644
index 0cf4ae4e..00000000
--- a/roles/monitoring/prometheus/server/templates/jobs/nut/ups.j2
+++ /dev/null
@@ -1,10 +0,0 @@
-  - job_name: '{{ job }}'
-    metrics_path: /nut/ups
-    scheme: https
-    tls_config:
-      ca_file: /etc/ssl/prometheus/ca-crt.pem
-      cert_file: /etc/ssl/prometheus/server/scrape-crt.pem
-      key_file: /etc/ssl/prometheus/server/scrape-key.pem
-    file_sd_configs:
-    - files:
-      - "/etc/prometheus/targets/{{ job }}/*.yml"
diff --git a/roles/monitoring/prometheus/server/templates/targets/blackbox/https.yml.j2 b/roles/monitoring/prometheus/server/templates/targets/blackbox/https.yml.j2
deleted file mode 100644
index 29c89590..00000000
--- a/roles/monitoring/prometheus/server/templates/targets/blackbox/https.yml.j2
+++ /dev/null
@@ -1,4 +0,0 @@
-- targets: [ '{{ hostvars[target.config.exporter_hostname].prometheus_scrape_endpoint }}' ]
-  labels:
-    instance: '{{ target.instance }}'
-    __param_target: '{{ target.config.address }}'
diff --git a/roles/monitoring/prometheus/server/templates/targets/blackbox/ping.yml.j2 b/roles/monitoring/prometheus/server/templates/targets/blackbox/ping.yml.j2
deleted file mode 100644
index 29c89590..00000000
--- a/roles/monitoring/prometheus/server/templates/targets/blackbox/ping.yml.j2
+++ /dev/null
@@ -1,4 +0,0 @@
-- targets: [ '{{ hostvars[target.config.exporter_hostname].prometheus_scrape_endpoint }}' ]
-  labels:
-    instance: '{{ target.instance }}'
-    __param_target: '{{ target.config.address }}'
diff --git a/roles/monitoring/prometheus/server/templates/targets/blackbox/probe.yml.j2 b/roles/monitoring/prometheus/server/templates/targets/blackbox/probe.yml.j2
new file mode 100644
index 00000000..4e336873
--- /dev/null
+++ b/roles/monitoring/prometheus/server/templates/targets/blackbox/probe.yml.j2
@@ -0,0 +1,5 @@
+- targets: [ '{{ hostvars[target.exporter_hostname].prometheus_scrape_endpoint }}' ]
+  labels:
+    instance: '{{ target.instance }}'
+    __param_target: '{{ target.config.target }}'
+    __param_module: '{{ target.config.module }}'
diff --git a/roles/monitoring/prometheus/server/templates/targets/blackbox/ssh.yml.j2 b/roles/monitoring/prometheus/server/templates/targets/blackbox/ssh.yml.j2
deleted file mode 100644
index 29c89590..00000000
--- a/roles/monitoring/prometheus/server/templates/targets/blackbox/ssh.yml.j2
+++ /dev/null
@@ -1,4 +0,0 @@
-- targets: [ '{{ hostvars[target.config.exporter_hostname].prometheus_scrape_endpoint }}' ]
-  labels:
-    instance: '{{ target.instance }}'
-    __param_target: '{{ target.config.address }}'
diff --git a/roles/monitoring/prometheus/server/templates/targets/nut/ups.yml.j2 b/roles/monitoring/prometheus/server/templates/targets/nut/ups.yml.j2
index 6003cd46..c60077c7 100644
--- a/roles/monitoring/prometheus/server/templates/targets/nut/ups.yml.j2
+++ b/roles/monitoring/prometheus/server/templates/targets/nut/ups.yml.j2
@@ -1,4 +1,4 @@
-- targets: [ '{{ hostvars[target.config.exporter_hostname].prometheus_scrape_endpoint }}' ]
+- targets: [ '{{ hostvars[target.exporter_hostname].prometheus_scrape_endpoint }}' ]
   labels:
     instance: '{{ target.instance }}'
     __param_ups: '{{ target.config.ups }}'
-- 
cgit v1.2.3