From c36e7b7a8f2dfe1c54e537b737340e025fa81467 Mon Sep 17 00:00:00 2001
From: Christian Pointner <equinox@spreadspace.org>
Date: Fri, 24 Sep 2021 02:04:31 +0200
Subject: add some alert rule for newly added node exporter metrics

---
 .../prometheus/server/defaults/main/rules_node.yml | 27 ++++++++++++++++++++++
 1 file changed, 27 insertions(+)

(limited to 'roles/monitoring/prometheus/server/defaults/main/rules_node.yml')

diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_node.yml b/roles/monitoring/prometheus/server/defaults/main/rules_node.yml
index ab7317ac..2c7f9319 100644
--- a/roles/monitoring/prometheus/server/defaults/main/rules_node.yml
+++ b/roles/monitoring/prometheus/server/defaults/main/rules_node.yml
@@ -217,3 +217,30 @@ prometheus_server_rules_node:
     annotations:
       summary: Host clock not synchronising (instance {{ '{{' }} $labels.instance {{ '}}' }})
       description: "Clock not synchronising.\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: AptUpgradesPending
+    expr: sum by (instance) (apt_upgrades_pending) > 0
+    for: 0m
+    labels:
+      severity: warning
+    annotations:
+      summary: Host has upgradeable packages (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "Host {{ '{{' }} $labels.instance {{ '}}' }} has {{ '{{' }} $value {{ '}}' }} upgradable packages.\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: AptAutoremovePending
+    expr: sum by (instance) (apt_autoremove_pending) > 0
+    for: 0m
+    labels:
+      severity: warning
+    annotations:
+      summary: Host has packages that can be autoremoved (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "Host {{ '{{' }} $labels.instance {{ '}}' }} has {{ '{{' }} $value {{ '}}' }} that can be autoremoved.\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: HostNeedsRebooting
+    expr: node_reboot_required > 0
+    for: 0m
+    labels:
+      severity: warning
+    annotations:
+      summary: Host must be rebootet (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "Host {{ '{{' }} $labels.instance {{ '}}' }} must be rebootet for security uppdates to take effect.\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
-- 
cgit v1.2.3


From 4ec26b272ab6090498a9eefa4a0efb06248b1ef4 Mon Sep 17 00:00:00 2001
From: Christian Pointner <equinox@spreadspace.org>
Date: Fri, 24 Sep 2021 14:11:35 +0200
Subject: prometheus: add monitoring for systemd units

---
 .../prometheus/exporter/node/defaults/main.yml       |  1 +
 .../prometheus/server/defaults/main/rules_node.yml   | 11 ++++++++++-
 .../prometheus/server/templates/jobs/node.j2         | 20 ++++++++++++++++++++
 3 files changed, 31 insertions(+), 1 deletion(-)
 create mode 100644 roles/monitoring/prometheus/server/templates/jobs/node.j2

(limited to 'roles/monitoring/prometheus/server/defaults/main/rules_node.yml')

diff --git a/roles/monitoring/prometheus/exporter/node/defaults/main.yml b/roles/monitoring/prometheus/exporter/node/defaults/main.yml
index 2714a7fe..491e70f6 100644
--- a/roles/monitoring/prometheus/exporter/node/defaults/main.yml
+++ b/roles/monitoring/prometheus/exporter/node/defaults/main.yml
@@ -12,6 +12,7 @@ prometheus_exporter_node_timesync_collector: "{{ _prometheus_exporter_node_time_
 prometheus_exporter_node_disable_collectors: []
 prometheus_exporter_node_extra_collectors:
   - "{{ prometheus_exporter_node_timesync_collector }}"
+  - systemd
 
 prometheus_exporter_node_textfile_collector_scripts:
   - deleted-libraries
diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_node.yml b/roles/monitoring/prometheus/server/defaults/main/rules_node.yml
index 2c7f9319..64a7d562 100644
--- a/roles/monitoring/prometheus/server/defaults/main/rules_node.yml
+++ b/roles/monitoring/prometheus/server/defaults/main/rules_node.yml
@@ -92,6 +92,15 @@ prometheus_server_rules_node:
       summary: Host CPU steal noisy neighbor (instance {{ '{{' }} $labels.instance {{ '}}' }})
       description: "CPU steal is > 10%. A noisy neighbor is killing VM performances or a spot instance may be out of credit.\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
 
+  - alert: HostSystemdNotRunning
+    expr: node_systemd_system_running == 0
+    for: 0m
+    labels:
+      severity: warning
+    annotations:
+      summary: Host systemd is not in running state (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "systemd is not in running state.\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
   - alert: HostSystemdServiceCrashed
     expr: node_systemd_unit_state{state="failed"} == 1
     for: 0m
@@ -99,7 +108,7 @@ prometheus_server_rules_node:
       severity: warning
     annotations:
       summary: Host systemd service crashed (instance {{ '{{' }} $labels.instance {{ '}}' }})
-      description: "systemd service crashed\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+      description: "The systemd service unit {{ '{{' }} $labels.name {{ '}}' }} is in failed state.\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
 
   - alert: HostPhysicalComponentTooHot
     expr: node_hwmon_temp_celsius > 75
diff --git a/roles/monitoring/prometheus/server/templates/jobs/node.j2 b/roles/monitoring/prometheus/server/templates/jobs/node.j2
new file mode 100644
index 00000000..ba9eab31
--- /dev/null
+++ b/roles/monitoring/prometheus/server/templates/jobs/node.j2
@@ -0,0 +1,20 @@
+  - job_name: '{{ job }}'
+    metrics_path: /proxy
+    params:
+      module:
+      - {{ job }}
+    scheme: https
+    tls_config:
+      ca_file: /etc/ssl/prometheus/ca-crt.pem
+      cert_file: /etc/ssl/prometheus/server/scrape-crt.pem
+      key_file: /etc/ssl/prometheus/server/scrape-key.pem
+    file_sd_configs:
+    - files:
+      - "/etc/prometheus/targets/{{ job }}/*.yml"
+    metric_relabel_configs:
+    - source_labels: [ "mountpoint" ]
+      regex: ".*/\\.snapshot/.*"
+      action: drop
+    - source_labels: [ "__name__", "state" ]
+      regex: "node_systemd_unit_state;(activating|deactivating|inactive)"
+      action: drop
-- 
cgit v1.2.3


From 93afffd62ab0da48230985440aae11afbe4de79b Mon Sep 17 00:00:00 2001
From: Christian Pointner <equinox@spreadspace.org>
Date: Sun, 26 Sep 2021 04:21:56 +0200
Subject: fix alert wording

---
 roles/monitoring/prometheus/server/defaults/main/rules_node.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'roles/monitoring/prometheus/server/defaults/main/rules_node.yml')

diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_node.yml b/roles/monitoring/prometheus/server/defaults/main/rules_node.yml
index 64a7d562..55641534 100644
--- a/roles/monitoring/prometheus/server/defaults/main/rules_node.yml
+++ b/roles/monitoring/prometheus/server/defaults/main/rules_node.yml
@@ -243,7 +243,7 @@ prometheus_server_rules_node:
       severity: warning
     annotations:
       summary: Host has packages that can be autoremoved (instance {{ '{{' }} $labels.instance {{ '}}' }})
-      description: "Host {{ '{{' }} $labels.instance {{ '}}' }} has {{ '{{' }} $value {{ '}}' }} that can be autoremoved.\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+      description: "Host {{ '{{' }} $labels.instance {{ '}}' }} has {{ '{{' }} $value {{ '}}' }} packages that can be autoremoved.\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
 
   - alert: HostNeedsRebooting
     expr: node_reboot_required > 0
-- 
cgit v1.2.3