revamp prometheus metric lables for whawty-nginx-ssotopic/promethues-multitarget-exporter-labels

5 files changed, 63 insertions, 29 deletions

diff --git a/inventory/host_vars/ch-http-proxy.yml b/inventory/host_vars/ch-http-proxy.yml
index 1e998ed0..024c9bfc 100644
--- a/inventory/host_vars/ch-http-proxy.yml
+++ b/inventory/host_vars/ch-http-proxy.yml
@@ -126,5 +126,4 @@ whawty_nginx_sso_logins:
 
 prometheus_job_multitarget_whawty_nginx_sso:
   ch-http-proxy:
-  - instance: "whawty-nginx-sso-{{ inventory_hostname }}-chaos-at-home"
-    instance_name: chaos-at-home
+  - app_instance: chaos-at-home
diff --git a/inventory/host_vars/ch-mon.yml b/inventory/host_vars/ch-mon.yml
index c514550e..e974b0af 100644
--- a/inventory/host_vars/ch-mon.yml
+++ b/inventory/host_vars/ch-mon.yml
@@ -121,8 +121,7 @@ whawty_nginx_sso_auths:
 
 prometheus_job_multitarget_whawty_nginx_sso:
   ch-mon:
-  - instance: "whawty-nginx-sso-{{ inventory_hostname }}-chaos-at-home"
-    instance_name: chaos-at-home
+  - app_instance: chaos-at-home
 
 
 prometheus_server_storage:
@@ -282,7 +281,7 @@ grafana_dashboards:
    datasource: "Prometheus"
  - file: standalone-kubelet-overview
    datasource: "Prometheus"
- - file: whawty-nginx-sso
+ - file: apps/whawty-nginx-sso
    datasource: "Prometheus"
  - file: mosquitto
    datasource: "Prometheus"
diff --git a/roles/monitoring/grafana/files/dashboard-whawty-nginx-sso.json b/roles/monitoring/grafana/files/dashboard-apps/whawty-nginx-sso.json
index b6f21f92..734893a5 100644
--- a/roles/monitoring/grafana/files/dashboard-whawty-nginx-sso.json
+++ b/roles/monitoring/grafana/files/dashboard-apps/whawty-nginx-sso.json
@@ -15,7 +15,7 @@
       "type": "grafana",
       "id": "grafana",
       "name": "Grafana",
-      "version": "10.2.2"
+      "version": "11.0.0"
     },
     {
       "type": "datasource",
@@ -146,6 +146,7 @@
           "showLegend": true
         },
         "tooltip": {
+          "maxHeight": 600,
           "mode": "multi",
           "sort": "none"
         }
@@ -246,6 +247,7 @@
           "showLegend": true
         },
         "tooltip": {
+          "maxHeight": 600,
           "mode": "single",
           "sort": "none"
         }
@@ -349,6 +351,7 @@
           "showLegend": true
         },
         "tooltip": {
+          "maxHeight": 600,
           "mode": "multi",
           "sort": "asc"
         }
@@ -378,7 +381,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 32
+        "y": 14
       },
       "id": 5,
       "panels": [],
@@ -448,7 +451,7 @@
         "h": 7,
         "w": 12,
         "x": 0,
-        "y": 33
+        "y": 15
       },
       "id": 13,
       "options": {
@@ -459,6 +462,7 @@
           "showLegend": true
         },
         "tooltip": {
+          "maxHeight": 600,
           "mode": "single",
           "sort": "none"
         }
@@ -579,7 +583,7 @@
         "h": 7,
         "w": 12,
         "x": 12,
-        "y": 33
+        "y": 15
       },
       "id": 14,
       "options": {
@@ -590,6 +594,7 @@
           "showLegend": true
         },
         "tooltip": {
+          "maxHeight": 600,
           "mode": "single",
           "sort": "none"
         }
@@ -710,7 +715,7 @@
         "h": 6,
         "w": 12,
         "x": 0,
-        "y": 40
+        "y": 22
       },
       "id": 21,
       "options": {
@@ -721,6 +726,7 @@
           "showLegend": true
         },
         "tooltip": {
+          "maxHeight": 600,
           "mode": "single",
           "sort": "none"
         }
@@ -811,7 +817,7 @@
         "h": 6,
         "w": 12,
         "x": 12,
-        "y": 40
+        "y": 22
       },
       "id": 22,
       "options": {
@@ -822,6 +828,7 @@
           "showLegend": true
         },
         "tooltip": {
+          "maxHeight": 600,
           "mode": "multi",
           "sort": "asc"
         }
@@ -850,7 +857,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 46
+        "y": 28
       },
       "id": 2,
       "panels": [],
@@ -951,7 +958,7 @@
         "h": 7,
         "w": 19,
         "x": 0,
-        "y": 47
+        "y": 29
       },
       "id": 1,
       "options": {
@@ -962,6 +969,7 @@
           "showLegend": true
         },
         "tooltip": {
+          "maxHeight": 600,
           "mode": "single",
           "sort": "none"
         }
@@ -986,7 +994,6 @@
         }
       ],
       "title": "Requests",
-      "transformations": [],
       "type": "timeseries"
     },
     {
@@ -1047,7 +1054,7 @@
         "h": 7,
         "w": 5,
         "x": 19,
-        "y": 47
+        "y": 29
       },
       "id": 3,
       "options": {
@@ -1062,10 +1069,11 @@
           "fields": "",
           "values": false
         },
+        "showPercentChange": false,
         "textMode": "auto",
         "wideLayout": true
       },
-      "pluginVersion": "10.2.2",
+      "pluginVersion": "11.0.0",
       "targets": [
         {
           "datasource": {
@@ -1198,7 +1206,7 @@
         "h": 6,
         "w": 12,
         "x": 0,
-        "y": 54
+        "y": 36
       },
       "id": 23,
       "options": {
@@ -1209,6 +1217,7 @@
           "showLegend": true
         },
         "tooltip": {
+          "maxHeight": 600,
           "mode": "single",
           "sort": "none"
         }
@@ -1299,7 +1308,7 @@
         "h": 6,
         "w": 12,
         "x": 12,
-        "y": 54
+        "y": 36
       },
       "id": 24,
       "options": {
@@ -1310,6 +1319,7 @@
           "showLegend": true
         },
         "tooltip": {
+          "maxHeight": 600,
           "mode": "multi",
           "sort": "asc"
         }
@@ -1426,7 +1436,7 @@
         "h": 6,
         "w": 12,
         "x": 0,
-        "y": 60
+        "y": 42
       },
       "id": 25,
       "options": {
@@ -1437,6 +1447,7 @@
           "showLegend": true
         },
         "tooltip": {
+          "maxHeight": 600,
           "mode": "single",
           "sort": "none"
         }
@@ -1529,7 +1540,7 @@
         "h": 6,
         "w": 12,
         "x": 12,
-        "y": 60
+        "y": 42
       },
       "id": 26,
       "options": {
@@ -1540,6 +1551,7 @@
           "showLegend": true
         },
         "tooltip": {
+          "maxHeight": 600,
           "mode": "multi",
           "sort": "asc"
         }
@@ -1569,7 +1581,7 @@
     }
   ],
   "refresh": "",
-  "schemaVersion": 38,
+  "schemaVersion": 39,
   "tags": [],
   "templating": {
     "list": [
@@ -1579,7 +1591,7 @@
           "type": "prometheus",
           "uid": "${DS_PROMETHEUS}"
         },
-        "definition": "label_values({job=\"whawty-nginx-sso\"},instance)",
+        "definition": "label_values({app_kind=\"whawty-nginx-sso\"},instance)",
         "hide": 0,
         "includeAll": false,
         "multi": false,
@@ -1587,7 +1599,27 @@
         "options": [],
         "query": {
           "qryType": 1,
-          "query": "label_values({job=\"whawty-nginx-sso\"},instance)",
+          "query": "label_values({app_kind=\"whawty-nginx-sso\"},instance)",
+          "refId": "PrometheusVariableQueryEditor-VariableQuery"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "type": "query"
+      },
+      {
+        "current": {},
+        "definition": "label_values({instance=\"$instance\"},app_instance)",
+        "hide": 0,
+        "includeAll": true,
+        "label": "",
+        "multi": true,
+        "name": "app_instance",
+        "options": [],
+        "query": {
+          "qryType": 1,
+          "query": "label_values({instance=\"$instance\"},app_instance)",
           "refId": "PrometheusVariableQueryEditor-VariableQuery"
         },
         "refresh": 1,
@@ -1602,7 +1634,7 @@
           "type": "prometheus",
           "uid": "${DS_PROMETHEUS}"
         },
-        "definition": "label_values(whawty_nginx_sso_web_requests_total{instance=\"$instance\"},endpoint)",
+        "definition": "label_values(whawty_nginx_sso_web_requests_total{instance=\"$instance\", app_instance=~\"$app_instance\"},endpoint)",
         "hide": 0,
         "includeAll": true,
         "multi": true,
@@ -1610,7 +1642,7 @@
         "options": [],
         "query": {
           "qryType": 1,
-          "query": "label_values(whawty_nginx_sso_web_requests_total{instance=\"$instance\"},endpoint)",
+          "query": "label_values(whawty_nginx_sso_web_requests_total{instance=\"$instance\", app_instance=~\"$app_instance\"},endpoint)",
           "refId": "PrometheusVariableQueryEditor-VariableQuery"
         },
         "refresh": 2,
@@ -1625,7 +1657,7 @@
           "type": "prometheus",
           "uid": "${DS_PROMETHEUS}"
         },
-        "definition": "label_values(whawty_nginx_sso_auth_ldap_requests_total{instance=\"$instance\"},server)",
+        "definition": "label_values(whawty_nginx_sso_auth_ldap_requests_total{instance=\"$instance\", app_instance=~\"$app_instance\"},server)",
         "hide": 0,
         "includeAll": true,
         "multi": true,
@@ -1633,7 +1665,7 @@
         "options": [],
         "query": {
           "qryType": 1,
-          "query": "label_values(whawty_nginx_sso_auth_ldap_requests_total{instance=\"$instance\"},server)",
+          "query": "label_values(whawty_nginx_sso_auth_ldap_requests_total{instance=\"$instance\", app_instance=~\"$app_instance\"},server)",
           "refId": "PrometheusVariableQueryEditor-VariableQuery"
         },
         "refresh": 2,
@@ -1648,10 +1680,11 @@
     "from": "now-24h",
     "to": "now"
   },
+  "timeRangeUpdatedDuringEditOrView": false,
   "timepicker": {},
   "timezone": "",
   "title": "whawty nginx-sso",
   "uid": "e745f2f9-9115-49a0-b386-9919bcf566de",
-  "version": 2,
+  "version": 15,
   "weekStart": ""
 }
\ No newline at end of file
diff --git a/roles/monitoring/prometheus/exporter/register/filter_plugins/prometheus.py b/roles/monitoring/prometheus/exporter/register/filter_plugins/prometheus.py
index 9c99113d..57b69a13 100644
--- a/roles/monitoring/prometheus/exporter/register/filter_plugins/prometheus.py
+++ b/roles/monitoring/prometheus/exporter/register/filter_plugins/prometheus.py
@@ -8,6 +8,7 @@ from ansible import errors
 
 _multitarget_id_map_ = {
     "blackbox/probe": (lambda vars, config: (vars['inventory_hostname'], "%s-%s" % (config['svc_kind'], config['svc_instance']))),
+    "whawty-nginx-sso": (lambda vars, config: (vars['inventory_hostname'], "%s-%s" % (config['app_instance'], vars['inventory_hostname']))),
 }
 
 
diff --git a/roles/monitoring/prometheus/exporter/register/templates/whawty-nginx-sso.yml.j2 b/roles/monitoring/prometheus/exporter/register/templates/whawty-nginx-sso.yml.j2
index 74ada64e..8675a143 100644
--- a/roles/monitoring/prometheus/exporter/register/templates/whawty-nginx-sso.yml.j2
+++ b/roles/monitoring/prometheus/exporter/register/templates/whawty-nginx-sso.yml.j2
@@ -1,7 +1,9 @@
 - targets: [ '{{ (target.exporter_hostname == prometheus_server) | ternary('127.0.0.1:9999', hostvars[target.exporter_hostname].prometheus_scrape_endpoint) }}' ]
   labels:
     instance: '{{ target.instance }}'
-    __metrics_path__: '/whawty-nginx-sso/{{ target.config.instance_name }}'
+    app_kind: 'whawty-nginx-sso'
+    app_instance: '{{ target.config.app_instance }}'
+    __metrics_path__: '/whawty-nginx-sso/{{ target.config.app_instance }}'
 {% for name, value in prometheus_target_labels.items() %}
     {{ name }}: '{{ value }}'
 {% endfor %}