Merge branch 'topic/promethues-multitarget-exporter-labels'

23 files changed, 78 insertions, 49 deletions

diff --git a/inventory/group_vars/chaos-at-home-ups/vars.yml b/inventory/group_vars/chaos-at-home-ups/vars.yml
index 31be4572..5c0de008 100644
--- a/inventory/group_vars/chaos-at-home-ups/vars.yml
+++ b/inventory/group_vars/chaos-at-home-ups/vars.yml
@@ -16,6 +16,5 @@ prometheus_exporters_default:
 
 prometheus_job_multitarget_nut__ups:
   ch-mon:
-  - instance: "ups-{{ ups_name }}"
-    ups: "{{ ups_name }}"
+  - ups: "{{ ups_name }}"
     server: "{{ network_mgmt_zone.prefix | ansible.utils.ipaddr(network_mgmt_zone.offsets[inventory_hostname]) | ansible.utils.ipaddr('address') }}"
diff --git a/inventory/group_vars/ele-ups/vars.yml b/inventory/group_vars/ele-ups/vars.yml
index 6146bc37..7d99ac83 100644
--- a/inventory/group_vars/ele-ups/vars.yml
+++ b/inventory/group_vars/ele-ups/vars.yml
@@ -20,6 +20,5 @@ prometheus_exporters_default:
 
 prometheus_job_multitarget_nut__ups:
   ele-calypso:
-  - instance: "ups-{{ ups_name }}"
-    ups: "{{ ups_name }}"
+  - ups: "{{ ups_name }}"
     server: "{{ network_mgmt_zone.prefix | ansible.utils.ipaddr(network_mgmt_zone.offsets[inventory_hostname]) | ansible.utils.ipaddr('address') }}"
diff --git a/inventory/host_vars/ch-apps/vars.yml b/inventory/host_vars/ch-apps/vars.yml
index 89a6873c..f39d57f1 100644
--- a/inventory/host_vars/ch-apps/vars.yml
+++ b/inventory/host_vars/ch-apps/vars.yml
@@ -65,21 +65,22 @@ prometheus_exporters_extra:
 
 prometheus_job_multitarget_blackbox__probe:
   ch-mon:
-  - instance: "ssh-{{ inventory_hostname }}"
+  - svc_kind: ssh
+    svc_instance: "{{ inventory_hostname }}"
     target: "{{ network_zones.svc.prefix | ansible.utils.ipaddr(network_zones.svc.offsets[inventory_hostname]) | ansible.utils.ipaddr('address') }}:{{ ansible_port | default(22) }}"
     module: ssh_banner
 
 prometheus_job_multitarget_ssl__probe:
   ch-apps:
-  - instance: "sslcert-standalone-kubelet-{{ inventory_hostname }}"
+  - module: file
     target: "/etc/ssl/standalone-kubelet/*.pem"
-    module: file
-  - instance: "sslcert-node-red-{{ inventory_hostname }}"
+    sslcert_instance: "standalone-kubelet"
+  - module: file
     target: "/etc/ssl/node-red-*/*.pem"
-    module: file
-  - instance: "sslcert-whawty-auth-{{ inventory_hostname }}"
+    sslcert_instance: "node-red"
+  - module: file
     target: "/etc/ssl/whawty-auth-*/*.pem"
-    module: file
+    sslcert_instance: "whawty-auth"
 
 
 zfs_arc_size:
diff --git a/inventory/host_vars/ch-atlas.yml b/inventory/host_vars/ch-atlas.yml
index f342445b..1f4dda2f 100644
--- a/inventory/host_vars/ch-atlas.yml
+++ b/inventory/host_vars/ch-atlas.yml
@@ -73,6 +73,7 @@ prometheus_exporter_node_textfile_collector_scripts:
 
 prometheus_job_multitarget_blackbox__probe:
   ch-mon:
-  - instance: "ssh-{{ inventory_hostname }}"
+  - svc_kind: ssh
+    svc_instance: "{{ inventory_hostname }}"
     target: "{{ network.primary.address | ansible.utils.ipaddr('address') }}:{{ ansible_port | default(22) }}"
     module: ssh_banner
diff --git a/inventory/host_vars/ch-epimetheus.yml b/inventory/host_vars/ch-epimetheus.yml
index 15e5f622..39ddbc3d 100644
--- a/inventory/host_vars/ch-epimetheus.yml
+++ b/inventory/host_vars/ch-epimetheus.yml
@@ -40,7 +40,8 @@ prometheus_exporter_node_textfile_collector_scripts:
 
 prometheus_job_multitarget_blackbox__probe:
   ch-mon:
-  - instance: "ssh-{{ inventory_hostname }}"
+  - svc_kind: ssh
+    svc_instance: "{{ inventory_hostname }}"
     target: "{{ network_zones.lan.prefix | ansible.utils.ipaddr(network_zones.lan.offsets[inventory_hostname]) | ansible.utils.ipaddr('address') }}:{{ ansible_port | default(22) }}"
     module: ssh_banner
 
diff --git a/inventory/host_vars/ch-greenbone.yml b/inventory/host_vars/ch-greenbone.yml
index 05489600..e5e4c3fc 100644
--- a/inventory/host_vars/ch-greenbone.yml
+++ b/inventory/host_vars/ch-greenbone.yml
@@ -43,10 +43,12 @@ spreadspace_apt_repo_components:
 
 prometheus_job_multitarget_blackbox__probe:
   ch-mon:
-  - instance: "ssh-{{ inventory_hostname }}"
+  - svc_kind: ssh
+    svc_instance: "{{ inventory_hostname }}"
     target: "{{ network_zones.svc.prefix | ansible.utils.ipaddr(network_zones.svc.offsets[inventory_hostname]) | ansible.utils.ipaddr('address') }}:{{ ansible_port | default(22) }}"
     module: ssh_banner
-  - instance: "https-greenbone.chaos-at-home.org"
+  - svc_kind: https
+    svc_instance: "greenbone.chaos-at-home.org"
     target: "https://{{ network_zones.svc.prefix | ansible.utils.ipaddr(network_zones.svc.offsets[inventory_hostname]) | ansible.utils.ipaddr('address') }}/robots.txt"
     module: http_tls_2xx
 
diff --git a/inventory/host_vars/ch-gw-lan.yml b/inventory/host_vars/ch-gw-lan.yml
index 11bc30e0..5677359c 100644
--- a/inventory/host_vars/ch-gw-lan.yml
+++ b/inventory/host_vars/ch-gw-lan.yml
@@ -48,7 +48,8 @@ spreadspace_apt_repo_components:
 
 prometheus_job_multitarget_blackbox__probe:
   ch-mon:
-  - instance: "ssh-{{ inventory_hostname }}"
+  - svc_kind: ssh
+    svc_instance: "{{ inventory_hostname }}"
     target: "{{ network_zones.svc.prefix | ansible.utils.ipaddr(network_zones.svc.offsets[inventory_hostname]) | ansible.utils.ipaddr('address') }}:{{ ansible_port | default(22) }}"
     module: ssh_banner
 
diff --git a/inventory/host_vars/ch-http-proxy.yml b/inventory/host_vars/ch-http-proxy.yml
index 53c3cfce..bdbde798 100644
--- a/inventory/host_vars/ch-http-proxy.yml
+++ b/inventory/host_vars/ch-http-proxy.yml
@@ -49,19 +49,21 @@ prometheus_exporters_extra:
 
 prometheus_job_multitarget_blackbox__probe:
   ch-mon:
-  - instance: "ssh-{{ inventory_hostname }}"
+  - svc_kind: ssh
+    svc_instance: "{{ inventory_hostname }}"
     target: "{{ network_zones.svc.prefix | ansible.utils.ipaddr(network_zones.svc.offsets[inventory_hostname]) | ansible.utils.ipaddr('address') }}:{{ ansible_port | default(22) }}"
     module: ssh_banner
-  - instance: "https-login.chaos-at-home.org"
+  - svc_kind: https
+    svc_instance: "login.chaos-at-home.org"
     target: "https://{{ network_services.http.addr }}/login"
     module: "http_tls_2xx"
     hostname: "login.chaos-at-home.org"
 
 prometheus_job_multitarget_ssl__probe:
   ch-http-proxy:
-  - instance: "sslcert-apps-publish-{{ inventory_hostname }}"
+  - module: file
     target: "/etc/ssl/apps-publish-*/*.pem"
-    module: file
+    sslcert_instance: apps-publish
 
 
 whawty_auth_store_instances:
@@ -124,5 +126,4 @@ whawty_nginx_sso_logins:
 
 prometheus_job_multitarget_whawty_nginx_sso:
   ch-http-proxy:
-  - instance: "whawty-nginx-sso-{{ inventory_hostname }}-chaos-at-home"
-    instance_name: chaos-at-home
+  - app_instance: chaos-at-home
diff --git a/inventory/host_vars/ch-imap-proxy.yml b/inventory/host_vars/ch-imap-proxy.yml
index 76a62757..b54fabcc 100644
--- a/inventory/host_vars/ch-imap-proxy.yml
+++ b/inventory/host_vars/ch-imap-proxy.yml
@@ -42,7 +42,8 @@ spreadspace_apt_repo_components:
 
 prometheus_job_multitarget_blackbox__probe:
   ch-mon:
-  - instance: "ssh-{{ inventory_hostname }}"
+  - svc_kind: ssh
+    svc_instance: "{{ inventory_hostname }}"
     target: "{{ network_zones.svc.prefix | ansible.utils.ipaddr(network_zones.svc.offsets[inventory_hostname]) | ansible.utils.ipaddr('address') }}:{{ ansible_port | default(22) }}"
     module: ssh_banner
 
diff --git a/inventory/host_vars/ch-iot.yml b/inventory/host_vars/ch-iot.yml
index 1a5782b4..30bee5d8 100644
--- a/inventory/host_vars/ch-iot.yml
+++ b/inventory/host_vars/ch-iot.yml
@@ -92,10 +92,12 @@ prometheus_exporters_extra:
 
 prometheus_job_multitarget_blackbox__probe:
   ch-mon:
-  - instance: "ssh-{{ inventory_hostname }}"
+  - svc_kind: ssh
+    svc_instance: "{{ inventory_hostname }}"
     target: "{{ network_zones.svc.prefix | ansible.utils.ipaddr(network_zones.svc.offsets[inventory_hostname]) | ansible.utils.ipaddr('address') }}:{{ ansible_port | default(22) }}"
     module: ssh_banner
-  - instance: "mqtt-mqtt.chaos-at-home.org"
+  - svc_kind: mqtt
+    svc_instance: "mqtt.chaos-at-home.org"
     target: "{{ network_zones.svc.prefix | ansible.utils.ipaddr(network_zones.svc.offsets[inventory_hostname]) | ansible.utils.ipaddr('address') }}:1883"
     module: "tcp_tls_connect"
     hostname: "mqtt.chaos-at-home.org"
diff --git a/inventory/host_vars/ch-jump.yml b/inventory/host_vars/ch-jump.yml
index 92adb5cf..d0d84bf5 100644
--- a/inventory/host_vars/ch-jump.yml
+++ b/inventory/host_vars/ch-jump.yml
@@ -68,6 +68,7 @@ spreadspace_apt_repo_components:
 
 prometheus_job_multitarget_blackbox__probe:
   ch-mon:
-  - instance: "ssh-{{ inventory_hostname }}"
+  - svc_kind: ssh
+    svc_instance: "{{ inventory_hostname }}"
     target: "{{ network_zones.svc.prefix | ansible.utils.ipaddr(network_zones.svc.offsets[inventory_hostname]) | ansible.utils.ipaddr('address') }}:{{ ansible_port | default(22) }}"
     module: ssh_banner
diff --git a/inventory/host_vars/ch-mimas.yml b/inventory/host_vars/ch-mimas.yml
index baf60b5e..bc09509d 100644
--- a/inventory/host_vars/ch-mimas.yml
+++ b/inventory/host_vars/ch-mimas.yml
@@ -134,10 +134,12 @@ prometheus_exporters_extra:
 
 prometheus_job_multitarget_blackbox__probe:
   ch-mon:
-  - instance: "ssh-{{ inventory_hostname }}"
+  - svc_kind: ssh
+    svc_instance: "{{ inventory_hostname }}"
     target: "{{ external_ip }}:{{ ansible_port | default(22) }}"
     module: ssh_banner
-  - instance: "https-mimas.chaos-at-home.org"
+  - svc_kind: https
+    svc_instance: "mimas.chaos-at-home.org"
     target: "https://mimas.chaos-at-home.org"
     module: http_tls_2xx
 
diff --git a/inventory/host_vars/ch-mon.yml b/inventory/host_vars/ch-mon.yml
index 64121848..85b2dd8c 100644
--- a/inventory/host_vars/ch-mon.yml
+++ b/inventory/host_vars/ch-mon.yml
@@ -121,8 +121,7 @@ whawty_nginx_sso_auths:
 
 prometheus_job_multitarget_whawty_nginx_sso:
   ch-mon:
-  - instance: "whawty-nginx-sso-{{ inventory_hostname }}-chaos-at-home"
-    instance_name: chaos-at-home
+  - app_instance: chaos-at-home
 
 
 prometheus_server_storage:
@@ -174,18 +173,21 @@ prometheus_exporter_smokeping_targets:
 
 prometheus_job_multitarget_blackbox__probe:
   ch-mon:
-  - instance: "ssh-{{ inventory_hostname }}"
+  - svc_kind: ssh
+    svc_instance: "{{ inventory_hostname }}"
     target: "{{ network_zones.svc.prefix | ansible.utils.ipaddr(network_zones.svc.offsets[inventory_hostname]) | ansible.utils.ipaddr('address') }}:{{ ansible_port | default(22) }}"
     module: ssh_banner
-  - instance: "https-mon.chaos-at-home.org"
+  - svc_kind: https
+    svc_instance: "mon.chaos-at-home.org"
     target: "https://{{ network_zones.svc.prefix | ansible.utils.ipaddr(network_zones.svc.offsets[inventory_hostname]) | ansible.utils.ipaddr('address') }}/healthz"
     module: http_tls_2xx
 
 prometheus_job_multitarget_ssl__probe:
   ch-mon:
-  - instance: "sslcert-prometheus-{{ inventory_hostname }}"
+  - module: file
     target: "/etc/ssl/prometheus/**/*.pem"
-    module: file
+    sslcert_instance: prometheus
+
 
 prometheus_server_rules_node_extra:
   - alert: GitFsckMetricsOutdated
@@ -266,7 +268,11 @@ grafana_dashboards:
    datasource: "Prometheus"
  - file: environment-sensors
    datasource: "Prometheus"
- - file: blackbox
+ - file: blackbox/ssh
+   datasource: "Prometheus"
+ - file: blackbox/https
+   datasource: "Prometheus"
+ - file: blackbox/mqtt
    datasource: "Prometheus"
  - file: smokeping
    datasource: "Prometheus"
@@ -276,7 +282,7 @@ grafana_dashboards:
    datasource: "Prometheus"
  - file: standalone-kubelet-overview
    datasource: "Prometheus"
- - file: whawty-nginx-sso
+ - file: apps/whawty-nginx-sso
    datasource: "Prometheus"
  - file: mosquitto
    datasource: "Prometheus"
diff --git a/inventory/host_vars/ch-pan.yml b/inventory/host_vars/ch-pan.yml
index d8e17277..29ec85ae 100644
--- a/inventory/host_vars/ch-pan.yml
+++ b/inventory/host_vars/ch-pan.yml
@@ -170,10 +170,12 @@ prometheus_exporters_extra:
 
 prometheus_job_multitarget_blackbox__probe:
   ch-mon:
-  - instance: "ssh-{{ inventory_hostname }}"
+  - svc_kind: ssh
+    svc_instance: "{{ inventory_hostname }}"
     target: "{{ network.primary.address | ansible.utils.ipaddr('address') }}:{{ ansible_port | default(22) }}"
     module: ssh_banner
-  - instance: "https-pan.chaos-at-home.org"
+  - svc_kind: https
+    svc_instance: "pan.chaos-at-home.org"
     target: "https://pan.chaos-at-home.org"
     module: http_tls_2xx
 
diff --git a/inventory/host_vars/ch-phoebe.yml b/inventory/host_vars/ch-phoebe.yml
index cfcfebc8..0f2ed044 100644
--- a/inventory/host_vars/ch-phoebe.yml
+++ b/inventory/host_vars/ch-phoebe.yml
@@ -69,7 +69,8 @@ prometheus_exporter_ipmi_modules:
 
 prometheus_job_multitarget_blackbox__probe:
   ch-mon:
-  - instance: "ssh-{{ inventory_hostname }}"
+  - svc_kind: ssh
+    svc_instance: "{{ inventory_hostname }}"
     target: "{{ network_zones.lan.prefix | ansible.utils.ipaddr(network_zones.lan.offsets[inventory_hostname]) | ansible.utils.ipaddr('address') }}:{{ ansible_port | default(22) }}"
     module: ssh_banner
 
diff --git a/inventory/host_vars/ch-prometheus.yml b/inventory/host_vars/ch-prometheus.yml
index de7e273b..b5641464 100644
--- a/inventory/host_vars/ch-prometheus.yml
+++ b/inventory/host_vars/ch-prometheus.yml
@@ -64,7 +64,8 @@ prometheus_exporter_ipmi_modules:
 
 prometheus_job_multitarget_blackbox__probe:
   ch-mon:
-  - instance: "ssh-{{ inventory_hostname }}"
+  - svc_kind: ssh
+    svc_instance: "{{ inventory_hostname }}"
     target: "{{ network_zones.lan.prefix | ansible.utils.ipaddr(network_zones.lan.offsets[inventory_hostname]) | ansible.utils.ipaddr('address') }}:{{ ansible_port | default(22) }}"
     module: ssh_banner
 
diff --git a/inventory/host_vars/ch-repo.yml b/inventory/host_vars/ch-repo.yml
index 8ff107a5..de952d74 100644
--- a/inventory/host_vars/ch-repo.yml
+++ b/inventory/host_vars/ch-repo.yml
@@ -76,9 +76,11 @@ approx_backends:
 
 prometheus_job_multitarget_blackbox__probe:
   ch-mon:
-  - instance: "ssh-{{ inventory_hostname }}"
+  - svc_kind: ssh
+    svc_instance: "{{ inventory_hostname }}"
     target: "{{ network_zones.svc.prefix | ansible.utils.ipaddr(network_zones.svc.offsets[inventory_hostname]) | ansible.utils.ipaddr('address') }}:{{ ansible_port | default(22) }}"
     module: ssh_banner
-  - instance: "http-apt.chaos-at-home.org"
+  - svc_kind: http
+    svc_instance: "apt.chaos-at-home.org"
     target: "http://{{ approx_hostname }}"
     module: "http_2xx"
diff --git a/inventory/host_vars/ele-calypso.yml b/inventory/host_vars/ele-calypso.yml
index 2f5648ac..75054533 100644
--- a/inventory/host_vars/ele-calypso.yml
+++ b/inventory/host_vars/ele-calypso.yml
@@ -122,7 +122,8 @@ prometheus_exporter_blackbox_modules_extra:
 
 prometheus_job_multitarget_blackbox__probe:
   ele-calypso:
-  - instance: "ssh-{{ inventory_hostname }}"
+  - svc_kind: ssh
+    svc_instance: "{{ inventory_hostname }}"
     target: "{{ network_zones.lan.prefix | ansible.utils.ipaddr(network_zones.lan.offsets[inventory_hostname]) | ansible.utils.ipaddr('address') }}:{{ ansible_port | default(22) }}"
     module: ssh_banner
 
@@ -178,7 +179,7 @@ grafana_dashboards:
    datasource: "Prometheus"
  - file: chrony
    datasource: "Prometheus"
- - file: blackbox
+ - file: blackbox/ssh
    datasource: "Prometheus"
  - file: network-ups-tools
    datasource: "Prometheus"
diff --git a/inventory/host_vars/ele-dione.yml b/inventory/host_vars/ele-dione.yml
index 7b1d98d7..7f5b8a31 100644
--- a/inventory/host_vars/ele-dione.yml
+++ b/inventory/host_vars/ele-dione.yml
@@ -98,6 +98,7 @@ prometheus_exporter_ipmi_modules:
 
 prometheus_job_multitarget_blackbox__probe:
   ele-calypso:
-  - instance: "ssh-{{ inventory_hostname }}"
+  - svc_kind: ssh
+    svc_instance: "{{ inventory_hostname }}"
     target: "{{ network_zones.emc.prefix | ansible.utils.ipaddr(network_zones.emc.offsets[inventory_hostname]) | ansible.utils.ipaddr('address') }}:{{ ansible_port | default(22) }}"
     module: ssh_banner
diff --git a/inventory/host_vars/ele-helene.yml b/inventory/host_vars/ele-helene.yml
index 8d73ce80..48efb84b 100644
--- a/inventory/host_vars/ele-helene.yml
+++ b/inventory/host_vars/ele-helene.yml
@@ -66,7 +66,8 @@ prometheus_exporter_ipmi_modules:
 
 prometheus_job_multitarget_blackbox__probe:
   ele-calypso:
-  - instance: "ssh-{{ inventory_hostname }}"
+  - svc_kind: ssh
+    svc_instance: "{{ inventory_hostname }}"
     target: "{{ network_zones.lan.prefix | ansible.utils.ipaddr(network_zones.lan.offsets[inventory_hostname]) | ansible.utils.ipaddr('address') }}:{{ ansible_port | default(22) }}"
     module: ssh_banner
 
diff --git a/inventory/host_vars/ele-media.yml b/inventory/host_vars/ele-media.yml
index 09f08d63..286b6ff9 100644
--- a/inventory/host_vars/ele-media.yml
+++ b/inventory/host_vars/ele-media.yml
@@ -51,7 +51,8 @@ prometheus_exporter_node_textfile_collector_scripts:
 
 prometheus_job_multitarget_blackbox__probe:
   ele-calypso:
-  - instance: "ssh-{{ inventory_hostname }}"
+  - svc_kind: ssh
+    svc_instance: "{{ inventory_hostname }}"
     target: "{{ network_zones.lan.prefix | ansible.utils.ipaddr(network_zones.lan.offsets[inventory_hostname]) | ansible.utils.ipaddr('address') }}:{{ ansible_port | default(22) }}"
     module: ssh_banner
 
diff --git a/inventory/host_vars/ele-telesto.yml b/inventory/host_vars/ele-telesto.yml
index 030a7111..f70efa16 100644
--- a/inventory/host_vars/ele-telesto.yml
+++ b/inventory/host_vars/ele-telesto.yml
@@ -74,6 +74,7 @@ prometheus_exporter_ipmi_modules:
 
 prometheus_job_multitarget_blackbox__probe:
   ele-calypso:
-  - instance: "ssh-{{ inventory_hostname }}"
+  - svc_kind: ssh
+    svc_instance: "{{ inventory_hostname }}"
     target: "{{ network_zones.emc.prefix | ansible.utils.ipaddr(network_zones.emc.offsets[inventory_hostname]) | ansible.utils.ipaddr('address') }}:{{ ansible_port | default(22) }}"
     module: ssh_banner
diff --git a/inventory/host_vars/ele-thetys.yml b/inventory/host_vars/ele-thetys.yml
index 2144327c..bd72f98a 100644
--- a/inventory/host_vars/ele-thetys.yml
+++ b/inventory/host_vars/ele-thetys.yml
@@ -51,7 +51,8 @@ prometheus_exporters_extra:
 
 prometheus_job_multitarget_blackbox__probe:
   ele-calypso:
-  - instance: "ssh-{{ inventory_hostname }}"
+  - svc_kind: ssh
+    svc_instance: "{{ inventory_hostname }}"
     target: "{{ network_zones.cc_orpheum.prefix | ansible.utils.ipaddr(network_zones.cc_orpheum.offsets[inventory_hostname]) | ansible.utils.ipaddr('address') }}:{{ ansible_port | default(22) }}"
     module: ssh_banner