prometheus: move blackbox job config to new setup

14 files changed, 94 insertions, 74 deletions

diff --git a/inventory/group_vars/chaos-at-home-ups/vars.yml b/inventory/group_vars/chaos-at-home-ups/vars.yml
index cc7ae585..5ff68452 100644
--- a/inventory/group_vars/chaos-at-home-ups/vars.yml
+++ b/inventory/group_vars/chaos-at-home-ups/vars.yml
@@ -12,7 +12,7 @@ prometheus_exporters_default:
   - openwrt
 
 prometheus_special_job_nut:
-  exporter_hostname: ch-mon
-  instance: "ups-{{ ups_name }}"
-  ups: "{{ ups_name }}"
-  server: "{{ network_mgmt_zone.prefix | ipaddr(network_mgmt_zone.offsets[inventory_hostname]) | ipaddr('address') }}"
+  - exporter_hostname: ch-mon
+    instance: "ups-{{ ups_name }}"
+    ups: "{{ ups_name }}"
+    server: "{{ network_mgmt_zone.prefix | ipaddr(network_mgmt_zone.offsets[inventory_hostname]) | ipaddr('address') }}"
diff --git a/inventory/group_vars/promzone-chaos-at-home/vars.yml b/inventory/group_vars/promzone-chaos-at-home/vars.yml
index a48cbad7..fcb04716 100644
--- a/inventory/group_vars/promzone-chaos-at-home/vars.yml
+++ b/inventory/group_vars/promzone-chaos-at-home/vars.yml
@@ -11,6 +11,9 @@ prometheus_server_jobs_generic:
   - openwrt
 prometheus_server_jobs_special:
   - nut
+  - blackbox-ping
+  - blackbox-https
+  - blackbox-ssh
 
 prometheus_zone_name: chaos@home
 prometheus_zone_targets: "{{ groups['promzone-chaos-at-home'] }}"
diff --git a/inventory/host_vars/ch-mon.yml b/inventory/host_vars/ch-mon.yml
index 1a050b6a..7d8e334b 100644
--- a/inventory/host_vars/ch-mon.yml
+++ b/inventory/host_vars/ch-mon.yml
@@ -76,6 +76,24 @@ prometheus_exporter_blackbox_modules_extra:
   icmp:
     prober: icmp
 
+prometheus_special_job_blackbox_ping:
+  - exporter_hostname: ch-mon
+    instance: "ping-magentagw"
+    address: 62.99.185.129
+  - exporter_hostname: ch-mon
+    instance: "ping-quad9"
+    address: 9.9.9.9
+
+prometheus_special_job_blackbox_https:
+  - exporter_hostname: ch-mon
+    instance: "https-web.chaos-at-home.org"
+    address: web.chaos-at-home.org
+
+prometheus_special_job_blackbox_ssh:
+  - exporter_hostname: ch-mon
+    instance: "ssh-{{ inventory_hostname }}"
+    address: "{{ network_zones.svc.prefix | ipaddr(network_zones.svc.offsets[inventory_hostname]) | ipaddr('address') }}:{{ ansible_port | default(22) }}"
+
 
 promethues_alertmanager_smtp:
   smarthost: "{{ network_zones.lan.prefix | ipaddr(network_zones.lan.offsets['ch-prometheus-legacy']) | ipaddr('address') }}:25"
@@ -86,71 +104,3 @@ prometheus_alertmanager_web_route_prefix: /alertmanager/
 
 
 grafana_secret_key: "{{ vault_grafana_secret_key }}"
-
-prometheus_server_jobs_extra: |
-  - job_name: 'ping'
-    metrics_path: /proxy
-    params:
-      module:
-      - blackbox
-      - icmp
-    scheme: https
-    tls_config:
-      ca_file: /etc/ssl/prometheus/ca-crt.pem
-      cert_file: /etc/ssl/prometheus/server/scrape-crt.pem
-      key_file: /etc/ssl/prometheus/server/scrape-key.pem
-    static_configs:
-      - targets:
-        - 62.99.185.129
-        - 9.9.9.9
-    relabel_configs:
-      - source_labels: [__address__]
-        target_label: __param_target
-      - source_labels: [__param_target]
-        target_label: instance
-      - target_label: __address__
-        replacement: 192.168.32.230:9999
-
-  - job_name: 'https'
-    metrics_path: /proxy
-    params:
-      module:
-      - blackbox
-      - http_tls_2xx
-    scheme: https
-    tls_config:
-      ca_file: /etc/ssl/prometheus/ca-crt.pem
-      cert_file: /etc/ssl/prometheus/server/scrape-crt.pem
-      key_file: /etc/ssl/prometheus/server/scrape-key.pem
-    static_configs:
-      - targets:
-        - web.chaos-at-home.org
-    relabel_configs:
-      - source_labels: [__address__]
-        target_label: __param_target
-      - source_labels: [__param_target]
-        target_label: instance
-      - target_label: __address__
-        replacement: 192.168.32.230:9999
-
-  - job_name: 'ssh'
-    metrics_path: /proxy
-    params:
-      module:
-      - blackbox
-      - ssh_banner
-    scheme: https
-    tls_config:
-      ca_file: /etc/ssl/prometheus/ca-crt.pem
-      cert_file: /etc/ssl/prometheus/server/scrape-crt.pem
-      key_file: /etc/ssl/prometheus/server/scrape-key.pem
-    static_configs:
-      - targets:
-        - 192.168.32.230:222
-    relabel_configs:
-      - source_labels: [__address__]
-        target_label: __param_target
-      - target_label: instance
-        replacement: 'ch-mon'
-      - target_label: __address__
-        replacement: 192.168.32.230:9999
diff --git a/roles/monitoring/prometheus/server/defaults/main/main.yml b/roles/monitoring/prometheus/server/defaults/main/main.yml
index f6679fa7..95b9da6d 100644
--- a/roles/monitoring/prometheus/server/defaults/main/main.yml
+++ b/roles/monitoring/prometheus/server/defaults/main/main.yml
@@ -16,6 +16,9 @@ prometheus_server_rules:
   node: "{{ prometheus_server_rules_node + prometheus_server_rules_node_extra }}"
   openwrt: "{{ prometheus_server_rules_openwrt + prometheus_server_rules_node_extra }}"
   nut: "{{ prometheus_server_rules_nut + prometheus_server_rules_nut_extra }}"
+  "blackbox-ping": "{{ prometheus_server_rules_blackbox_ping + prometheus_server_rules_blackbox_ping_extra }}"
+  "blackbox-https": "{{ prometheus_server_rules_blackbox_https + prometheus_server_rules_blackbox_https_extra }}"
+  "blackbox-ssh": "{{ prometheus_server_rules_blackbox_ssh + prometheus_server_rules_blackbox_ssh_extra }}"
 
 # prometheus_server_alertmanager:
 #   url: "127.0.0.1:9093"
diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_blackbox-https.yml b/roles/monitoring/prometheus/server/defaults/main/rules_blackbox-https.yml
new file mode 100644
index 00000000..bb806075
--- /dev/null
+++ b/roles/monitoring/prometheus/server/defaults/main/rules_blackbox-https.yml
@@ -0,0 +1,3 @@
+---
+prometheus_server_rules_blackbox_https_extra: []
+prometheus_server_rules_blackbox_https: []
diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_blackbox-ping.yml b/roles/monitoring/prometheus/server/defaults/main/rules_blackbox-ping.yml
new file mode 100644
index 00000000..56c122f5
--- /dev/null
+++ b/roles/monitoring/prometheus/server/defaults/main/rules_blackbox-ping.yml
@@ -0,0 +1,3 @@
+---
+prometheus_server_rules_blackbox_ping_extra: []
+prometheus_server_rules_blackbox_ping: []
diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_blackbox-ssh.yml b/roles/monitoring/prometheus/server/defaults/main/rules_blackbox-ssh.yml
new file mode 100644
index 00000000..727d2292
--- /dev/null
+++ b/roles/monitoring/prometheus/server/defaults/main/rules_blackbox-ssh.yml
@@ -0,0 +1,3 @@
+---
+prometheus_server_rules_blackbox_ssh_extra: []
+prometheus_server_rules_blackbox_ssh: []
diff --git a/roles/monitoring/prometheus/server/filter_plugins/prometheus.py b/roles/monitoring/prometheus/server/filter_plugins/prometheus.py
index 6e18481c..5a8722c2 100644
--- a/roles/monitoring/prometheus/server/filter_plugins/prometheus.py
+++ b/roles/monitoring/prometheus/server/filter_plugins/prometheus.py
@@ -23,9 +23,10 @@ def prometheus_special_job_targets(hostvars, jobs, targets):
         result = []
         for job in jobs:
             for target in targets:
-                config_varname = 'prometheus_special_job_' + job
+                config_varname = 'prometheus_special_job_' + job.replace('-', '_')
                 if config_varname in hostvars[target]:
-                    result.append({'job': job, 'instance': hostvars[target][config_varname]['instance'], 'config': hostvars[target][config_varname]})
+                    for config in hostvars[target][config_varname]:
+                        result.append({'job': job, 'instance': config['instance'], 'config': config})
         return result
     except Exception as e:
         raise errors.AnsibleFilterError("prometheus_special_job_targets(): %s" % str(e))
diff --git a/roles/monitoring/prometheus/server/templates/job-snippets/blackbox-https.j2 b/roles/monitoring/prometheus/server/templates/job-snippets/blackbox-https.j2
new file mode 100644
index 00000000..0a6d2dfa
--- /dev/null
+++ b/roles/monitoring/prometheus/server/templates/job-snippets/blackbox-https.j2
@@ -0,0 +1,14 @@
+  - job_name: '{{ job }}'
+    metrics_path: /proxy
+    params:
+      module:
+      - blackbox
+      - http_tls_2xx
+    scheme: https
+    tls_config:
+      ca_file: /etc/ssl/prometheus/ca-crt.pem
+      cert_file: /etc/ssl/prometheus/server/scrape-crt.pem
+      key_file: /etc/ssl/prometheus/server/scrape-key.pem
+    file_sd_configs:
+    - files:
+      - "/etc/prometheus/jobs/{{ job }}/*.yml"
diff --git a/roles/monitoring/prometheus/server/templates/job-snippets/blackbox-ping.j2 b/roles/monitoring/prometheus/server/templates/job-snippets/blackbox-ping.j2
new file mode 100644
index 00000000..7f4f12df
--- /dev/null
+++ b/roles/monitoring/prometheus/server/templates/job-snippets/blackbox-ping.j2
@@ -0,0 +1,14 @@
+  - job_name: '{{ job }}'
+    metrics_path: /proxy
+    params:
+      module:
+      - blackbox
+      - icmp
+    scheme: https
+    tls_config:
+      ca_file: /etc/ssl/prometheus/ca-crt.pem
+      cert_file: /etc/ssl/prometheus/server/scrape-crt.pem
+      key_file: /etc/ssl/prometheus/server/scrape-key.pem
+    file_sd_configs:
+    - files:
+      - "/etc/prometheus/jobs/{{ job }}/*.yml"
diff --git a/roles/monitoring/prometheus/server/templates/job-snippets/blackbox-ssh.j2 b/roles/monitoring/prometheus/server/templates/job-snippets/blackbox-ssh.j2
new file mode 100644
index 00000000..18381e32
--- /dev/null
+++ b/roles/monitoring/prometheus/server/templates/job-snippets/blackbox-ssh.j2
@@ -0,0 +1,14 @@
+  - job_name: '{{ job }}'
+    metrics_path: /proxy
+    params:
+      module:
+      - blackbox
+      - ssh_banner
+    scheme: https
+    tls_config:
+      ca_file: /etc/ssl/prometheus/ca-crt.pem
+      cert_file: /etc/ssl/prometheus/server/scrape-crt.pem
+      key_file: /etc/ssl/prometheus/server/scrape-key.pem
+    file_sd_configs:
+    - files:
+      - "/etc/prometheus/jobs/{{ job }}/*.yml"
diff --git a/roles/monitoring/prometheus/server/templates/targets/blackbox-https.yml.j2 b/roles/monitoring/prometheus/server/templates/targets/blackbox-https.yml.j2
new file mode 100644
index 00000000..e843de36
--- /dev/null
+++ b/roles/monitoring/prometheus/server/templates/targets/blackbox-https.yml.j2
@@ -0,0 +1,4 @@
+- targets: [ "{{ hostvars[target.config.exporter_hostname].prometheus_scrape_endpoint }}" ]
+  labels:
+    instance: "{{ target.instance }}"
+    __param_target: {{ target.config.address }}
diff --git a/roles/monitoring/prometheus/server/templates/targets/blackbox-ping.yml.j2 b/roles/monitoring/prometheus/server/templates/targets/blackbox-ping.yml.j2
new file mode 100644
index 00000000..e843de36
--- /dev/null
+++ b/roles/monitoring/prometheus/server/templates/targets/blackbox-ping.yml.j2
@@ -0,0 +1,4 @@
+- targets: [ "{{ hostvars[target.config.exporter_hostname].prometheus_scrape_endpoint }}" ]
+  labels:
+    instance: "{{ target.instance }}"
+    __param_target: {{ target.config.address }}
diff --git a/roles/monitoring/prometheus/server/templates/targets/blackbox-ssh.yml.j2 b/roles/monitoring/prometheus/server/templates/targets/blackbox-ssh.yml.j2
new file mode 100644
index 00000000..e843de36
--- /dev/null
+++ b/roles/monitoring/prometheus/server/templates/targets/blackbox-ssh.yml.j2
@@ -0,0 +1,4 @@
+- targets: [ "{{ hostvars[target.config.exporter_hostname].prometheus_scrape_endpoint }}" ]
+  labels:
+    instance: "{{ target.instance }}"
+    __param_target: {{ target.config.address }}