monitoring/prometheus: add generic alert for outdated textfile metrics

1 files changed, 24 insertions, 28 deletions

diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_node.yml b/roles/monitoring/prometheus/server/defaults/main/rules_node.yml
index 2366221d..76571dc3 100644
--- a/roles/monitoring/prometheus/server/defaults/main/rules_node.yml
+++ b/roles/monitoring/prometheus/server/defaults/main/rules_node.yml
@@ -2,15 +2,6 @@
 ## https://awesome-prometheus-alerts.grep.to/rules#host-and-hardware
 prometheus_server_rules_node_extra: []
 prometheus_server_rules_node:
-  - alert: NodeTextfileScrapeFailed
-    expr: node_textfile_scrape_error != 0
-    for: 0m
-    labels:
-      severity: warning
-    annotations:
-      summary: Scraping metrics from textfiles failed (instance {{ '{{' }} $labels.instance {{ '}}' }})
-      description: "The textfile collector failed to scrape at least one file\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
-
   - alert: HostOutOfMemory
     expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100 < 10
     for: 2m
@@ -271,6 +262,26 @@ prometheus_server_rules_node:
       summary: ZFS zpool is faulted (instance {{ '{{' }} $labels.instance {{ '}}' }})
       description: "The ZFS zpool {{ '{{' }} $labels.zpool {{ '}}' }} is faulted.\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
 
+  ### textfile-collectors
+  - alert: NodeTextfileScrapeFailed
+    expr: node_textfile_scrape_error != 0
+    for: 0m
+    labels:
+      severity: warning
+    annotations:
+      summary: Scraping metrics from textfiles failed (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "The textfile collector failed to scrape at least one file\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: NodeTextfileMetricsOutdated
+    expr: (time() - node_textfile_mtime_seconds{file!~".*/(git-fsck-|syncoid-pull-).*.prom"}) > 3600
+    for: 0m
+    labels:
+      severity: warning
+    annotations:
+      summary: Metrics from a textfile collector are too old (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "The exported values from textfile {{ '{{' }} $labels.file {{ '}}' }} on host {{ '{{' }} $labels.instance {{ '}}' }} are {{ '{{' }} $value {{ '}}' }} seconds old.\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  ## textfile-collector: apt
   - alert: AptUpgradesPending
     expr: sum by (instance) (apt_upgrades_pending) > 0
     for: 0m
@@ -298,6 +309,7 @@ prometheus_server_rules_node:
       summary: Host must be rebootet (instance {{ '{{' }} $labels.instance {{ '}}' }})
       description: "Host {{ '{{' }} $labels.instance {{ '}}' }} must be rebootet for security uppdates to take effect.\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
 
+  ## textfile-collector: deleted-libraries
   - alert: DeletedLibraryInUse
     expr: node_processes_linking_deleted_libraries
     for: 2m
@@ -307,24 +319,7 @@ prometheus_server_rules_node:
       summary: Some processes still use a deleted library (instance {{ '{{' }} $labels.instance {{ '}}' }})
       description: "The deleted library {{ '{{' }} $labels.library_name {{ '}}' }} on host {{ '{{' }} $labels.instance {{ '}}' }} is still in use by {{ '{{' }} $value {{ '}}' }} processes.\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
 
-  - alert: SmartmonMetricsMissing
-    expr: absent(smartmon_smartctl_run)
-    for: 30m
-    labels:
-      severity: warning
-    annotations:
-      summary: Metrics from smartctl are missing (instance {{ '{{' }} $labels.instance {{ '}}' }})
-      description: "smartctl on host {{ '{{' }} $labels.instance {{ '}}' }} stopped reporting metrics for device {{ '{{' }} $labels.device {{ '}}' }}.\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
-
-  - alert: SmartmonMetricsOutdated
-    expr: time() - smartmon_smartctl_run > 3600
-    for: 0m
-    labels:
-      severity: warning
-    annotations:
-      summary: Metrics from smartctl are too old (instance {{ '{{' }} $labels.instance {{ '}}' }})
-      description: "The exported values from smartctl on host {{ '{{' }} $labels.instance {{ '}}' }} are {{ '{{' }} $value {{ '}}' }} seconds old.\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
-
+  ## textfile-collector: smartmon
   - alert: SmartmonDeviceUnhealthy
     expr: smartmon_device_smart_healthy == 0
     for: 0m
@@ -343,7 +338,7 @@ prometheus_server_rules_node:
       summary: Host disk reports S.M.A.R.T. errors (instance {{ '{{' }} $labels.instance {{ '}}' }})
       description: "S.M.A.R.T. reports errors for disk {{ '{{' }} $labels.device {{ '}}' }} on host {{ '{{' }} $labels.instance {{ '}}' }}.\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
 
-  ### selection of S.M.A.R.T. metrics based on: https://www.backblaze.com/blog/hard-drive-smart-stats/
+  # selection of S.M.A.R.T. metrics based on: https://www.backblaze.com/blog/hard-drive-smart-stats/
   - alert: SmartmonReallocatedSectorsCount
     expr: delta(smartmon_attr_raw_value{name="reallocated_sector_ct"}[72h]) > 0
     for: 0m
@@ -398,6 +393,7 @@ prometheus_server_rules_node:
       summary: Host disk S.M.A.R.T. reports offline uncorrectable errors (instance {{ '{{' }} $labels.instance {{ '}}' }})
       description: "S.M.A.R.T. reports offline uncorrectable errors for disk {{ '{{' }} $labels.device {{ '}}' }} on host {{ '{{' }} $labels.instance {{ '}}' }}, the drive might be failing.\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
 
+  ## syncoid-pull scripts
   - alert: SyncoidPullJobTooLongAgo
     expr: time() - syncoid_pull_run > (24 * 3600)
     for: 0m