From 67dbf646f686dac08419cad521a178a793be2c22 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Thu, 21 Sep 2023 18:37:34 +0200 Subject: fix smartctl text collector and add some alerts for nvme metrics --- .../prometheus/server/defaults/main/rules_node.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) (limited to 'roles/monitoring/prometheus/server/defaults') diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_node.yml b/roles/monitoring/prometheus/server/defaults/main/rules_node.yml index 5493da57..fe6e781b 100644 --- a/roles/monitoring/prometheus/server/defaults/main/rules_node.yml +++ b/roles/monitoring/prometheus/server/defaults/main/rules_node.yml @@ -393,6 +393,24 @@ prometheus_server_rules_node: summary: Host disk S.M.A.R.T. reports offline uncorrectable errors (instance {{ '{{' }} $labels.instance {{ '}}' }}) description: "S.M.A.R.T. reports offline uncorrectable errors for disk {{ '{{' }} $labels.device {{ '}}' }} on host {{ '{{' }} $labels.instance {{ '}}' }}, the drive might be failing.\n VALUE = {{ '{{' }} $value {{ '}}' }}\n LABELS = {{ '{{' }} $labels {{ '}}' }}" + - alert: SmartmonNVMEMediaErrors + expr: delta(smartmon_media_errors_total[72h]) > 0 + for: 0m + labels: + severity: warning + annotations: + summary: Host disk S.M.A.R.T. reports NVMe media errors. (instance {{ '{{' }} $labels.instance {{ '}}' }}) + description: "S.M.A.R.T. reports NVMe media errors for disk {{ '{{' }} $labels.device {{ '}}' }} on host {{ '{{' }} $labels.instance {{ '}}' }}, the drive might be failing.\n VALUE = {{ '{{' }} $value {{ '}}' }}\n LABELS = {{ '{{' }} $labels {{ '}}' }}" + + - alert: SmartmonNVMEPercentageUsed + expr: smartmon_percentage_used_ratio > 80 + for: 0m + labels: + severity: warning + annotations: + summary: Host disk S.M.A.R.T. reports worn out disk. (instance {{ '{{' }} $labels.instance {{ '}}' }}) + description: "S.M.A.R.T. reports NVMe high usage of device {{ '{{' }} $labels.device {{ '}}' }} on host {{ '{{' }} $labels.instance {{ '}}' }}, the drive might be completely worn out soon.\n VALUE = {{ '{{' }} $value {{ '}}' }}\n LABELS = {{ '{{' }} $labels {{ '}}' }}" + ## syncoid-pull scripts - alert: SyncoidPullJobTooLongAgo expr: time() - syncoid_pull_run > (24 * 3600) -- cgit v1.2.3