From b2488aea07dd4bd92ff085870782bc7b8d0a9a25 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Thu, 25 May 2023 22:43:58 +0200 Subject: monitoring/prometheus: add generic alert for outdated textfile metrics --- .../prometheus/server/defaults/main/rules_node.yml | 52 ++++++++++------------ 1 file changed, 24 insertions(+), 28 deletions(-) (limited to 'roles/monitoring/prometheus') diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_node.yml b/roles/monitoring/prometheus/server/defaults/main/rules_node.yml index 2366221d..76571dc3 100644 --- a/roles/monitoring/prometheus/server/defaults/main/rules_node.yml +++ b/roles/monitoring/prometheus/server/defaults/main/rules_node.yml @@ -2,15 +2,6 @@ ## https://awesome-prometheus-alerts.grep.to/rules#host-and-hardware prometheus_server_rules_node_extra: [] prometheus_server_rules_node: - - alert: NodeTextfileScrapeFailed - expr: node_textfile_scrape_error != 0 - for: 0m - labels: - severity: warning - annotations: - summary: Scraping metrics from textfiles failed (instance {{ '{{' }} $labels.instance {{ '}}' }}) - description: "The textfile collector failed to scrape at least one file\n VALUE = {{ '{{' }} $value {{ '}}' }}\n LABELS = {{ '{{' }} $labels {{ '}}' }}" - - alert: HostOutOfMemory expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100 < 10 for: 2m @@ -271,6 +262,26 @@ prometheus_server_rules_node: summary: ZFS zpool is faulted (instance {{ '{{' }} $labels.instance {{ '}}' }}) description: "The ZFS zpool {{ '{{' }} $labels.zpool {{ '}}' }} is faulted.\n VALUE = {{ '{{' }} $value {{ '}}' }}\n LABELS = {{ '{{' }} $labels {{ '}}' }}" + ### textfile-collectors + - alert: NodeTextfileScrapeFailed + expr: node_textfile_scrape_error != 0 + for: 0m + labels: + severity: warning + annotations: + summary: Scraping metrics from textfiles failed (instance {{ '{{' }} $labels.instance {{ '}}' }}) + description: "The textfile collector failed to scrape at least one file\n VALUE = {{ '{{' }} $value {{ '}}' }}\n LABELS = {{ '{{' }} $labels {{ '}}' }}" + + - alert: NodeTextfileMetricsOutdated + expr: (time() - node_textfile_mtime_seconds{file!~".*/(git-fsck-|syncoid-pull-).*.prom"}) > 3600 + for: 0m + labels: + severity: warning + annotations: + summary: Metrics from a textfile collector are too old (instance {{ '{{' }} $labels.instance {{ '}}' }}) + description: "The exported values from textfile {{ '{{' }} $labels.file {{ '}}' }} on host {{ '{{' }} $labels.instance {{ '}}' }} are {{ '{{' }} $value {{ '}}' }} seconds old.\n VALUE = {{ '{{' }} $value {{ '}}' }}\n LABELS = {{ '{{' }} $labels {{ '}}' }}" + + ## textfile-collector: apt - alert: AptUpgradesPending expr: sum by (instance) (apt_upgrades_pending) > 0 for: 0m @@ -298,6 +309,7 @@ prometheus_server_rules_node: summary: Host must be rebootet (instance {{ '{{' }} $labels.instance {{ '}}' }}) description: "Host {{ '{{' }} $labels.instance {{ '}}' }} must be rebootet for security uppdates to take effect.\n VALUE = {{ '{{' }} $value {{ '}}' }}\n LABELS = {{ '{{' }} $labels {{ '}}' }}" + ## textfile-collector: deleted-libraries - alert: DeletedLibraryInUse expr: node_processes_linking_deleted_libraries for: 2m @@ -307,24 +319,7 @@ prometheus_server_rules_node: summary: Some processes still use a deleted library (instance {{ '{{' }} $labels.instance {{ '}}' }}) description: "The deleted library {{ '{{' }} $labels.library_name {{ '}}' }} on host {{ '{{' }} $labels.instance {{ '}}' }} is still in use by {{ '{{' }} $value {{ '}}' }} processes.\n VALUE = {{ '{{' }} $value {{ '}}' }}\n LABELS = {{ '{{' }} $labels {{ '}}' }}" - - alert: SmartmonMetricsMissing - expr: absent(smartmon_smartctl_run) - for: 30m - labels: - severity: warning - annotations: - summary: Metrics from smartctl are missing (instance {{ '{{' }} $labels.instance {{ '}}' }}) - description: "smartctl on host {{ '{{' }} $labels.instance {{ '}}' }} stopped reporting metrics for device {{ '{{' }} $labels.device {{ '}}' }}.\n VALUE = {{ '{{' }} $value {{ '}}' }}\n LABELS = {{ '{{' }} $labels {{ '}}' }}" - - - alert: SmartmonMetricsOutdated - expr: time() - smartmon_smartctl_run > 3600 - for: 0m - labels: - severity: warning - annotations: - summary: Metrics from smartctl are too old (instance {{ '{{' }} $labels.instance {{ '}}' }}) - description: "The exported values from smartctl on host {{ '{{' }} $labels.instance {{ '}}' }} are {{ '{{' }} $value {{ '}}' }} seconds old.\n VALUE = {{ '{{' }} $value {{ '}}' }}\n LABELS = {{ '{{' }} $labels {{ '}}' }}" - + ## textfile-collector: smartmon - alert: SmartmonDeviceUnhealthy expr: smartmon_device_smart_healthy == 0 for: 0m @@ -343,7 +338,7 @@ prometheus_server_rules_node: summary: Host disk reports S.M.A.R.T. errors (instance {{ '{{' }} $labels.instance {{ '}}' }}) description: "S.M.A.R.T. reports errors for disk {{ '{{' }} $labels.device {{ '}}' }} on host {{ '{{' }} $labels.instance {{ '}}' }}.\n VALUE = {{ '{{' }} $value {{ '}}' }}\n LABELS = {{ '{{' }} $labels {{ '}}' }}" - ### selection of S.M.A.R.T. metrics based on: https://www.backblaze.com/blog/hard-drive-smart-stats/ + # selection of S.M.A.R.T. metrics based on: https://www.backblaze.com/blog/hard-drive-smart-stats/ - alert: SmartmonReallocatedSectorsCount expr: delta(smartmon_attr_raw_value{name="reallocated_sector_ct"}[72h]) > 0 for: 0m @@ -398,6 +393,7 @@ prometheus_server_rules_node: summary: Host disk S.M.A.R.T. reports offline uncorrectable errors (instance {{ '{{' }} $labels.instance {{ '}}' }}) description: "S.M.A.R.T. reports offline uncorrectable errors for disk {{ '{{' }} $labels.device {{ '}}' }} on host {{ '{{' }} $labels.instance {{ '}}' }}, the drive might be failing.\n VALUE = {{ '{{' }} $value {{ '}}' }}\n LABELS = {{ '{{' }} $labels {{ '}}' }}" + ## syncoid-pull scripts - alert: SyncoidPullJobTooLongAgo expr: time() - syncoid_pull_run > (24 * 3600) for: 0m -- cgit v1.2.3