From 67dbf646f686dac08419cad521a178a793be2c22 Mon Sep 17 00:00:00 2001
From: Christian Pointner <equinox@spreadspace.org>
Date: Thu, 21 Sep 2023 18:37:34 +0200
Subject: fix smartctl text collector and add some alerts for nvme metrics

---
 .../templates/textfile-collector-scripts/smartmon.j2   |  1 +
 .../prometheus/server/defaults/main/rules_node.yml     | 18 ++++++++++++++++++
 2 files changed, 19 insertions(+)

(limited to 'roles/monitoring/prometheus')

diff --git a/roles/monitoring/prometheus/exporter/node/templates/textfile-collector-scripts/smartmon.j2 b/roles/monitoring/prometheus/exporter/node/templates/textfile-collector-scripts/smartmon.j2
index c6bfba28..f564cc22 100644
--- a/roles/monitoring/prometheus/exporter/node/templates/textfile-collector-scripts/smartmon.j2
+++ b/roles/monitoring/prometheus/exporter/node/templates/textfile-collector-scripts/smartmon.j2
@@ -491,6 +491,7 @@ def collect_nvme_metrics(device):
     attribute_lines = attributes.strip().split('\n')[6:]
     for line in attribute_lines:
         label, value = line.split(':')
+        value = value.replace(',', '')
         if label == 'Available Spare':
             metrics['available_spare_ratio'].labels(
                 device.base_labels["device"], device.base_labels["disk"]
diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_node.yml b/roles/monitoring/prometheus/server/defaults/main/rules_node.yml
index 5493da57..fe6e781b 100644
--- a/roles/monitoring/prometheus/server/defaults/main/rules_node.yml
+++ b/roles/monitoring/prometheus/server/defaults/main/rules_node.yml
@@ -393,6 +393,24 @@ prometheus_server_rules_node:
       summary: Host disk S.M.A.R.T. reports offline uncorrectable errors (instance {{ '{{' }} $labels.instance {{ '}}' }})
       description: "S.M.A.R.T. reports offline uncorrectable errors for disk {{ '{{' }} $labels.device {{ '}}' }} on host {{ '{{' }} $labels.instance {{ '}}' }}, the drive might be failing.\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
 
+  - alert: SmartmonNVMEMediaErrors
+    expr: delta(smartmon_media_errors_total[72h]) > 0
+    for: 0m
+    labels:
+      severity: warning
+    annotations:
+      summary: Host disk S.M.A.R.T. reports NVMe media errors. (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "S.M.A.R.T. reports NVMe media errors for disk {{ '{{' }} $labels.device {{ '}}' }} on host {{ '{{' }} $labels.instance {{ '}}' }}, the drive might be failing.\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: SmartmonNVMEPercentageUsed
+    expr: smartmon_percentage_used_ratio > 80
+    for: 0m
+    labels:
+      severity: warning
+    annotations:
+      summary: Host disk S.M.A.R.T. reports worn out disk. (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "S.M.A.R.T. reports NVMe high usage of device {{ '{{' }} $labels.device {{ '}}' }} on host {{ '{{' }} $labels.instance {{ '}}' }}, the drive might be completely worn out soon.\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
   ## syncoid-pull scripts
   - alert: SyncoidPullJobTooLongAgo
     expr: time() - syncoid_pull_run > (24 * 3600)
-- 
cgit v1.2.3