prometheus: revamp labels for metrcis from blackbox exporter

18 files changed, 63 insertions, 29 deletions

diff --git a/inventory/host_vars/ch-apps/vars.yml b/inventory/host_vars/ch-apps/vars.yml
index 89a6873c..8ea12d8b 100644
--- a/inventory/host_vars/ch-apps/vars.yml
+++ b/inventory/host_vars/ch-apps/vars.yml
@@ -65,7 +65,8 @@ prometheus_exporters_extra:
 
 prometheus_job_multitarget_blackbox__probe:
   ch-mon:
-  - instance: "ssh-{{ inventory_hostname }}"
+  - svc_kind: ssh
+    svc_name: "{{ inventory_hostname }}"
     target: "{{ network_zones.svc.prefix | ansible.utils.ipaddr(network_zones.svc.offsets[inventory_hostname]) | ansible.utils.ipaddr('address') }}:{{ ansible_port | default(22) }}"
     module: ssh_banner
 
diff --git a/inventory/host_vars/ch-atlas.yml b/inventory/host_vars/ch-atlas.yml
index f342445b..3ee9f7ac 100644
--- a/inventory/host_vars/ch-atlas.yml
+++ b/inventory/host_vars/ch-atlas.yml
@@ -73,6 +73,7 @@ prometheus_exporter_node_textfile_collector_scripts:
 
 prometheus_job_multitarget_blackbox__probe:
   ch-mon:
-  - instance: "ssh-{{ inventory_hostname }}"
+  - svc_kind: ssh
+    svc_name: "{{ inventory_hostname }}"
     target: "{{ network.primary.address | ansible.utils.ipaddr('address') }}:{{ ansible_port | default(22) }}"
     module: ssh_banner
diff --git a/inventory/host_vars/ch-epimetheus.yml b/inventory/host_vars/ch-epimetheus.yml
index 15e5f622..e76d10ae 100644
--- a/inventory/host_vars/ch-epimetheus.yml
+++ b/inventory/host_vars/ch-epimetheus.yml
@@ -40,7 +40,8 @@ prometheus_exporter_node_textfile_collector_scripts:
 
 prometheus_job_multitarget_blackbox__probe:
   ch-mon:
-  - instance: "ssh-{{ inventory_hostname }}"
+  - svc_kind: ssh
+    svc_name: "{{ inventory_hostname }}"
     target: "{{ network_zones.lan.prefix | ansible.utils.ipaddr(network_zones.lan.offsets[inventory_hostname]) | ansible.utils.ipaddr('address') }}:{{ ansible_port | default(22) }}"
     module: ssh_banner
 
diff --git a/inventory/host_vars/ch-greenbone.yml b/inventory/host_vars/ch-greenbone.yml
index 05489600..4372ecd1 100644
--- a/inventory/host_vars/ch-greenbone.yml
+++ b/inventory/host_vars/ch-greenbone.yml
@@ -43,10 +43,12 @@ spreadspace_apt_repo_components:
 
 prometheus_job_multitarget_blackbox__probe:
   ch-mon:
-  - instance: "ssh-{{ inventory_hostname }}"
+  - svc_kind: ssh
+    svc_name: "{{ inventory_hostname }}"
     target: "{{ network_zones.svc.prefix | ansible.utils.ipaddr(network_zones.svc.offsets[inventory_hostname]) | ansible.utils.ipaddr('address') }}:{{ ansible_port | default(22) }}"
     module: ssh_banner
-  - instance: "https-greenbone.chaos-at-home.org"
+  - svc_kind: https
+    svc_name: "greenbone.chaos-at-home.org"
     target: "https://{{ network_zones.svc.prefix | ansible.utils.ipaddr(network_zones.svc.offsets[inventory_hostname]) | ansible.utils.ipaddr('address') }}/robots.txt"
     module: http_tls_2xx
 
diff --git a/inventory/host_vars/ch-gw-lan.yml b/inventory/host_vars/ch-gw-lan.yml
index 11bc30e0..e621648c 100644
--- a/inventory/host_vars/ch-gw-lan.yml
+++ b/inventory/host_vars/ch-gw-lan.yml
@@ -48,7 +48,8 @@ spreadspace_apt_repo_components:
 
 prometheus_job_multitarget_blackbox__probe:
   ch-mon:
-  - instance: "ssh-{{ inventory_hostname }}"
+  - svc_kind: ssh
+    svc_name: "{{ inventory_hostname }}"
     target: "{{ network_zones.svc.prefix | ansible.utils.ipaddr(network_zones.svc.offsets[inventory_hostname]) | ansible.utils.ipaddr('address') }}:{{ ansible_port | default(22) }}"
     module: ssh_banner
 
diff --git a/inventory/host_vars/ch-http-proxy.yml b/inventory/host_vars/ch-http-proxy.yml
index 53c3cfce..a61eb68a 100644
--- a/inventory/host_vars/ch-http-proxy.yml
+++ b/inventory/host_vars/ch-http-proxy.yml
@@ -49,10 +49,12 @@ prometheus_exporters_extra:
 
 prometheus_job_multitarget_blackbox__probe:
   ch-mon:
-  - instance: "ssh-{{ inventory_hostname }}"
+  - svc_kind: ssh
+    svc_name: "{{ inventory_hostname }}"
     target: "{{ network_zones.svc.prefix | ansible.utils.ipaddr(network_zones.svc.offsets[inventory_hostname]) | ansible.utils.ipaddr('address') }}:{{ ansible_port | default(22) }}"
     module: ssh_banner
-  - instance: "https-login.chaos-at-home.org"
+  - svc_kind: https
+    svc_name: "login.chaos-at-home.org"
     target: "https://{{ network_services.http.addr }}/login"
     module: "http_tls_2xx"
     hostname: "login.chaos-at-home.org"
diff --git a/inventory/host_vars/ch-imap-proxy.yml b/inventory/host_vars/ch-imap-proxy.yml
index 76a62757..16c5e047 100644
--- a/inventory/host_vars/ch-imap-proxy.yml
+++ b/inventory/host_vars/ch-imap-proxy.yml
@@ -42,7 +42,8 @@ spreadspace_apt_repo_components:
 
 prometheus_job_multitarget_blackbox__probe:
   ch-mon:
-  - instance: "ssh-{{ inventory_hostname }}"
+  - svc_kind: ssh
+    svc_name: "{{ inventory_hostname }}"
     target: "{{ network_zones.svc.prefix | ansible.utils.ipaddr(network_zones.svc.offsets[inventory_hostname]) | ansible.utils.ipaddr('address') }}:{{ ansible_port | default(22) }}"
     module: ssh_banner
 
diff --git a/inventory/host_vars/ch-iot.yml b/inventory/host_vars/ch-iot.yml
index 1a5782b4..53af0fb5 100644
--- a/inventory/host_vars/ch-iot.yml
+++ b/inventory/host_vars/ch-iot.yml
@@ -92,10 +92,12 @@ prometheus_exporters_extra:
 
 prometheus_job_multitarget_blackbox__probe:
   ch-mon:
-  - instance: "ssh-{{ inventory_hostname }}"
+  - svc_kind: ssh
+    svc_name: "{{ inventory_hostname }}"
     target: "{{ network_zones.svc.prefix | ansible.utils.ipaddr(network_zones.svc.offsets[inventory_hostname]) | ansible.utils.ipaddr('address') }}:{{ ansible_port | default(22) }}"
     module: ssh_banner
-  - instance: "mqtt-mqtt.chaos-at-home.org"
+  - svc_kind: mqtt
+    svc_name: "mqtt.chaos-at-home.org"
     target: "{{ network_zones.svc.prefix | ansible.utils.ipaddr(network_zones.svc.offsets[inventory_hostname]) | ansible.utils.ipaddr('address') }}:1883"
     module: "tcp_tls_connect"
     hostname: "mqtt.chaos-at-home.org"
diff --git a/inventory/host_vars/ch-jump.yml b/inventory/host_vars/ch-jump.yml
index 92adb5cf..9af98aa1 100644
--- a/inventory/host_vars/ch-jump.yml
+++ b/inventory/host_vars/ch-jump.yml
@@ -68,6 +68,7 @@ spreadspace_apt_repo_components:
 
 prometheus_job_multitarget_blackbox__probe:
   ch-mon:
-  - instance: "ssh-{{ inventory_hostname }}"
+  - svc_kind: ssh
+    svc_name: "{{ inventory_hostname }}"
     target: "{{ network_zones.svc.prefix | ansible.utils.ipaddr(network_zones.svc.offsets[inventory_hostname]) | ansible.utils.ipaddr('address') }}:{{ ansible_port | default(22) }}"
     module: ssh_banner
diff --git a/inventory/host_vars/ch-mimas.yml b/inventory/host_vars/ch-mimas.yml
index baf60b5e..131d9396 100644
--- a/inventory/host_vars/ch-mimas.yml
+++ b/inventory/host_vars/ch-mimas.yml
@@ -134,10 +134,12 @@ prometheus_exporters_extra:
 
 prometheus_job_multitarget_blackbox__probe:
   ch-mon:
-  - instance: "ssh-{{ inventory_hostname }}"
+  - svc_kind: ssh
+    svc_name: "{{ inventory_hostname }}"
     target: "{{ external_ip }}:{{ ansible_port | default(22) }}"
     module: ssh_banner
-  - instance: "https-mimas.chaos-at-home.org"
+  - svc_kind: https
+    svc_name: "mimas.chaos-at-home.org"
     target: "https://mimas.chaos-at-home.org"
     module: http_tls_2xx
 
diff --git a/inventory/host_vars/ch-mon.yml b/inventory/host_vars/ch-mon.yml
index 64121848..4f5371ab 100644
--- a/inventory/host_vars/ch-mon.yml
+++ b/inventory/host_vars/ch-mon.yml
@@ -174,10 +174,12 @@ prometheus_exporter_smokeping_targets:
 
 prometheus_job_multitarget_blackbox__probe:
   ch-mon:
-  - instance: "ssh-{{ inventory_hostname }}"
+  - svc_kind: ssh
+    svc_name: "{{ inventory_hostname }}"
     target: "{{ network_zones.svc.prefix | ansible.utils.ipaddr(network_zones.svc.offsets[inventory_hostname]) | ansible.utils.ipaddr('address') }}:{{ ansible_port | default(22) }}"
     module: ssh_banner
-  - instance: "https-mon.chaos-at-home.org"
+  - svc_kind: https
+    svc_name: "mon.chaos-at-home.org"
     target: "https://{{ network_zones.svc.prefix | ansible.utils.ipaddr(network_zones.svc.offsets[inventory_hostname]) | ansible.utils.ipaddr('address') }}/healthz"
     module: http_tls_2xx
 
diff --git a/inventory/host_vars/ch-pan.yml b/inventory/host_vars/ch-pan.yml
index d8e17277..0edb86a1 100644
--- a/inventory/host_vars/ch-pan.yml
+++ b/inventory/host_vars/ch-pan.yml
@@ -170,10 +170,12 @@ prometheus_exporters_extra:
 
 prometheus_job_multitarget_blackbox__probe:
   ch-mon:
-  - instance: "ssh-{{ inventory_hostname }}"
+  - svc_kind: ssh
+    svc_name: "{{ inventory_hostname }}"
     target: "{{ network.primary.address | ansible.utils.ipaddr('address') }}:{{ ansible_port | default(22) }}"
     module: ssh_banner
-  - instance: "https-pan.chaos-at-home.org"
+  - svc_kind: https
+    svc_name: "pan.chaos-at-home.org"
     target: "https://pan.chaos-at-home.org"
     module: http_tls_2xx
 
diff --git a/inventory/host_vars/ch-phoebe.yml b/inventory/host_vars/ch-phoebe.yml
index cfcfebc8..ca787492 100644
--- a/inventory/host_vars/ch-phoebe.yml
+++ b/inventory/host_vars/ch-phoebe.yml
@@ -69,7 +69,8 @@ prometheus_exporter_ipmi_modules:
 
 prometheus_job_multitarget_blackbox__probe:
   ch-mon:
-  - instance: "ssh-{{ inventory_hostname }}"
+  - svc_kind: ssh
+    svc_name: "{{ inventory_hostname }}"
     target: "{{ network_zones.lan.prefix | ansible.utils.ipaddr(network_zones.lan.offsets[inventory_hostname]) | ansible.utils.ipaddr('address') }}:{{ ansible_port | default(22) }}"
     module: ssh_banner
 
diff --git a/inventory/host_vars/ch-prometheus.yml b/inventory/host_vars/ch-prometheus.yml
index de7e273b..df6fa39a 100644
--- a/inventory/host_vars/ch-prometheus.yml
+++ b/inventory/host_vars/ch-prometheus.yml
@@ -64,7 +64,8 @@ prometheus_exporter_ipmi_modules:
 
 prometheus_job_multitarget_blackbox__probe:
   ch-mon:
-  - instance: "ssh-{{ inventory_hostname }}"
+  - svc_kind: ssh
+    svc_name: "{{ inventory_hostname }}"
     target: "{{ network_zones.lan.prefix | ansible.utils.ipaddr(network_zones.lan.offsets[inventory_hostname]) | ansible.utils.ipaddr('address') }}:{{ ansible_port | default(22) }}"
     module: ssh_banner
 
diff --git a/inventory/host_vars/ch-repo.yml b/inventory/host_vars/ch-repo.yml
index 8ff107a5..c911195b 100644
--- a/inventory/host_vars/ch-repo.yml
+++ b/inventory/host_vars/ch-repo.yml
@@ -76,9 +76,11 @@ approx_backends:
 
 prometheus_job_multitarget_blackbox__probe:
   ch-mon:
-  - instance: "ssh-{{ inventory_hostname }}"
+  - svc_kind: ssh
+    svc_name: "{{ inventory_hostname }}"
     target: "{{ network_zones.svc.prefix | ansible.utils.ipaddr(network_zones.svc.offsets[inventory_hostname]) | ansible.utils.ipaddr('address') }}:{{ ansible_port | default(22) }}"
     module: ssh_banner
-  - instance: "http-apt.chaos-at-home.org"
+  - svc_kind: http
+    svc_name: "apt.chaos-at-home.org"
     target: "http://{{ approx_hostname }}"
     module: "http_2xx"
diff --git a/roles/monitoring/prometheus/exporter/register/filter_plugins/prometheus.py b/roles/monitoring/prometheus/exporter/register/filter_plugins/prometheus.py
index e8067a72..27ba29e6 100644
--- a/roles/monitoring/prometheus/exporter/register/filter_plugins/prometheus.py
+++ b/roles/monitoring/prometheus/exporter/register/filter_plugins/prometheus.py
@@ -6,6 +6,11 @@ from functools import partial
 from ansible import errors
 
 
+_multitarget_id_map_ = {
+    "blackbox/probe": (lambda vars, config: (vars['inventory_hostname'], "%s-%s" % (config['svc_kind'], config['svc_name']))),
+}
+
+
 def prometheus_job_targets(jobs, vars):
     try:
         result = []
@@ -14,15 +19,20 @@ def prometheus_job_targets(jobs, vars):
             if multitarget_config_varname in vars:
                 for exporter_hostname, configs in vars[multitarget_config_varname].items():
                     for config in configs:
-                        result.append({'job': job, 'instance': config['instance'], 'enabled': True,
+                        if job in _multitarget_id_map_:
+                            instance, id = _multitarget_id_map_[job](vars, config)
+                        else:
+                            id = config['instance']
+                            instance = config['instance']
+                        result.append({'_id': id, 'job': job, 'instance': instance, 'enabled': True,
                                        'exporter_hostname': exporter_hostname, 'config': config})
 
             else:
                 enabled = job in vars['prometheus_exporters_default'] or job in vars['prometheus_exporters_extra']
-                result.append({'job': job, 'instance': vars['inventory_hostname'], 'enabled': enabled})
+                result.append({'_id': vars['inventory_hostname'], 'job': job, 'instance': vars['inventory_hostname'], 'enabled': enabled})
         return result
     except Exception as e:
-        raise errors.AnsibleFilterError("prometheus_job_targets(): %s" % str(e))
+        raise errors.AnsibleFilterError("prometheus_job_targets(): %s" % repr(e))
 
 
 class FilterModule(object):
diff --git a/roles/monitoring/prometheus/exporter/register/tasks/main.yml b/roles/monitoring/prometheus/exporter/register/tasks/main.yml
index 519975d6..1d8585e9 100644
--- a/roles/monitoring/prometheus/exporter/register/tasks/main.yml
+++ b/roles/monitoring/prometheus/exporter/register/tasks/main.yml
@@ -8,20 +8,20 @@
     loop: "{{ job_targets | selectattr('enabled') }}"
     loop_control:
       loop_var: target
-      label: "{{ target.job }} -> {{ target.instance }}"
+      label: "{{ target.job }} -> {{ target._id }}"
     delegate_to: "{{ prometheus_server }}"
     template:
       src: "{{ lookup('first_found', {'paths': ['templates'], 'files': [target.job + '.yml.j2', 'generic.yml.j2']}) }}"
-      dest: "/etc/prometheus/targets/{{ target.job }}/{{ target.instance }}.yml"
+      dest: "/etc/prometheus/targets/{{ target.job }}/{{ target._id }}.yml"
     notify: reload prometheus
 
   - name: remove files for disabled targets
     loop: "{{ job_targets | rejectattr('enabled') }}"
     loop_control:
       loop_var: target
-      label: "{{ target.job }} -> {{ target.instance }}"
+      label: "{{ target.job }} -> {{ target._id }}"
     delegate_to: "{{ prometheus_server }}"
     file:
-      path: "/etc/prometheus/targets/{{ target.job }}/{{ target.instance }}.yml"
+      path: "/etc/prometheus/targets/{{ target.job }}/{{ target._id }}.yml"
       state: absent
     notify: reload prometheus
diff --git a/roles/monitoring/prometheus/exporter/register/templates/blackbox/probe.yml.j2 b/roles/monitoring/prometheus/exporter/register/templates/blackbox/probe.yml.j2
index 5f98e3a8..6319c1c5 100644
--- a/roles/monitoring/prometheus/exporter/register/templates/blackbox/probe.yml.j2
+++ b/roles/monitoring/prometheus/exporter/register/templates/blackbox/probe.yml.j2
@@ -1,6 +1,8 @@
 - targets: [ '{{ (target.exporter_hostname == prometheus_server) | ternary('127.0.0.1:9999', hostvars[target.exporter_hostname].prometheus_scrape_endpoint) }}' ]
   labels:
     instance: '{{ target.instance }}'
+    svc_kind: '{{ target.config.svc_kind }}'
+    svc_name: '{{ target.config.svc_name }}'
     __param_target: '{{ target.config.target }}'
     __param_module: '{{ target.config.module }}'
 {% if 'hostname' in target.config %}