prometheus/standalone-kubelet: simply config and scrape all sub-jobs on all hosts

7 files changed, 34 insertions, 22 deletions

diff --git a/inventory/group_vars/promzone-chaos-at-home/vars.yml b/inventory/group_vars/promzone-chaos-at-home/vars.yml
index 1cbd0208..b9268168 100644
--- a/inventory/group_vars/promzone-chaos-at-home/vars.yml
+++ b/inventory/group_vars/promzone-chaos-at-home/vars.yml
@@ -32,8 +32,6 @@ prometheus_server_jobs:
   - smokeping
   - bind
   - standalone-kubelet
-  - standalone-kubelet/resource
-  - standalone-kubelet/probes
 
 prometheus_zone_name: chaos@home
 
diff --git a/inventory/host_vars/ch-apps.yml b/inventory/host_vars/ch-apps.yml
index e3ea7492..3970469d 100644
--- a/inventory/host_vars/ch-apps.yml
+++ b/inventory/host_vars/ch-apps.yml
@@ -67,14 +67,6 @@ prometheus_job_multitarget_blackbox__probe:
     target: "{{ network_zones.svc.prefix | ansible.utils.ipaddr(network_zones.svc.offsets[inventory_hostname]) | ansible.utils.ipaddr('address') }}:{{ ansible_port | default(22) }}"
     module: ssh_banner
 
-prometheus_job_multitarget_standalone_kubelet__resource:
-  ch-apps:
-  - instance: "{{ inventory_hostname }}"
-
-prometheus_job_multitarget_standalone_kubelet__probes:
-  ch-apps:
-  - instance: "{{ inventory_hostname }}"
-
 
 zfs_arc_size:
   min: 512MB
diff --git a/roles/monitoring/grafana/files/dashboard-standalone-kubelet-overview.json b/roles/monitoring/grafana/files/dashboard-standalone-kubelet-overview.json
index 50003a81..5ff9a338 100644
--- a/roles/monitoring/grafana/files/dashboard-standalone-kubelet-overview.json
+++ b/roles/monitoring/grafana/files/dashboard-standalone-kubelet-overview.json
@@ -624,7 +624,7 @@
           "hide": false,
           "interval": "15s",
           "intervalFactor": 1,
-          "legendFormat": "{{ pod }}",
+          "legendFormat": "{{ instance }}",
           "metric": "container_cpu",
           "range": true,
           "refId": "A",
@@ -727,7 +727,7 @@
           "hide": false,
           "interval": "15s",
           "intervalFactor": 1,
-          "legendFormat": "{{ pod }}",
+          "legendFormat": "{{ instance }}",
           "metric": "container_cpu",
           "range": true,
           "refId": "A",
@@ -1576,6 +1576,10 @@
       },
       {
         "current": {},
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${DS_PROMETHEUS}"
+        },
         "definition": "label_values(pod_cpu_usage_seconds_total{instance=~\"$instance\"},pod)",
         "hide": 0,
         "includeAll": true,
@@ -1594,6 +1598,10 @@
       },
       {
         "current": {},
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${DS_PROMETHEUS}"
+        },
         "definition": "label_values(container_cpu_usage_seconds_total{instance=~\"$instance\", pod=~\"$pod\"},container)",
         "hide": 0,
         "includeAll": true,
@@ -1644,6 +1652,6 @@
   "timezone": "",
   "title": "Standalone Kubelet Overview",
   "uid": "3138fa155d5915769fbded898ac09fd9",
-  "version": 3,
+  "version": 4,
   "weekStart": ""
-}
\ No newline at end of file
+}
diff --git a/roles/monitoring/prometheus/server/defaults/main/main.yml b/roles/monitoring/prometheus/server/defaults/main/main.yml
index 21430a3d..1e0dcf32 100644
--- a/roles/monitoring/prometheus/server/defaults/main/main.yml
+++ b/roles/monitoring/prometheus/server/defaults/main/main.yml
@@ -36,8 +36,6 @@ prometheus_server_rules:
   snmp: "{{ prometheus_server_rules_snmp + prometheus_server_rules_snmp_extra }}"
   snmp/probe: "{{ prometheus_server_rules_snmp__probe + prometheus_server_rules_snmp__probe_extra }}"
   standalone-kubelet: "{{ prometheus_server_rules_standalone_kubelet + prometheus_server_rules_standalone_kubelet_extra }}"
-  standalone-kubelet/resource: "{{ prometheus_server_rules_standalone_kubelet__resource + prometheus_server_rules_standalone_kubelet__resource_extra }}"
-  standalone-kubelet/probes: "{{ prometheus_server_rules_standalone_kubelet__probes + prometheus_server_rules_standalone_kubelet__probes_extra }}"
   modbus: "{{ prometheus_server_rules_modbus + prometheus_server_rules_modbus_extra }}"
   modbus/probe: "{{ prometheus_server_rules_modbus__probe + prometheus_server_rules_modbus__probe_extra }}"
 
diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_standalone-kubelet__probes.yml b/roles/monitoring/prometheus/server/defaults/main/rules_standalone-kubelet__probes.yml
deleted file mode 100644
index a384045c..00000000
--- a/roles/monitoring/prometheus/server/defaults/main/rules_standalone-kubelet__probes.yml
+++ /dev/null
@@ -1,3 +0,0 @@
----
-prometheus_server_rules_standalone_kubelet__probes_extra: []
-prometheus_server_rules_standalone_kubelet__probes: []
diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_standalone-kubelet__resource.yml b/roles/monitoring/prometheus/server/defaults/main/rules_standalone-kubelet__resource.yml
deleted file mode 100644
index 6894e16f..00000000
--- a/roles/monitoring/prometheus/server/defaults/main/rules_standalone-kubelet__resource.yml
+++ /dev/null
@@ -1,3 +0,0 @@
----
-prometheus_server_rules_standalone_kubelet__resource_extra: []
-prometheus_server_rules_standalone_kubelet__resource: []
diff --git a/roles/monitoring/prometheus/server/templates/jobs/standalone-kubelet.j2 b/roles/monitoring/prometheus/server/templates/jobs/standalone-kubelet.j2
index eafd740a..77e7aeed 100644
--- a/roles/monitoring/prometheus/server/templates/jobs/standalone-kubelet.j2
+++ b/roles/monitoring/prometheus/server/templates/jobs/standalone-kubelet.j2
@@ -12,3 +12,25 @@
     - source_labels: [ "__name__" ]
       regex: "(kubernetes_feature_enabled|apiserver_.*)"
       action: drop
+
+  - job_name: '{{ job }}/resource'
+    metrics_path: /{{ job }}/resource
+    scheme: https
+    tls_config:
+      ca_file: /etc/ssl/prometheus/ca-crt.pem
+      cert_file: /etc/ssl/prometheus/server/scrape-crt.pem
+      key_file: /etc/ssl/prometheus/server/scrape-key.pem
+    file_sd_configs:
+    - files:
+      - "/etc/prometheus/targets/{{ job }}/*.yml"
+
+  - job_name: '{{ job }}/probes'
+    metrics_path: /{{ job }}/probes
+    scheme: https
+    tls_config:
+      ca_file: /etc/ssl/prometheus/ca-crt.pem
+      cert_file: /etc/ssl/prometheus/server/scrape-crt.pem
+      key_file: /etc/ssl/prometheus/server/scrape-key.pem
+    file_sd_configs:
+    - files:
+      - "/etc/prometheus/targets/{{ job }}/*.yml"