From b2ddd8aa0d7e5308a512d56630ef59a4b651c0dc Mon Sep 17 00:00:00 2001
From: Christian Pointner <equinox@spreadspace.org>
Date: Thu, 16 Dec 2021 19:55:48 +0100
Subject: prometheus: improve smartmon alerts

---
 .../monitoring/prometheus/server/defaults/main/rules_node.yml  | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

(limited to 'roles/monitoring')

diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_node.yml b/roles/monitoring/prometheus/server/defaults/main/rules_node.yml
index 0a28871d..e6415bd9 100644
--- a/roles/monitoring/prometheus/server/defaults/main/rules_node.yml
+++ b/roles/monitoring/prometheus/server/defaults/main/rules_node.yml
@@ -192,7 +192,7 @@ prometheus_server_rules_node:
       description: "Host {{ '{{' }} $labels.instance {{ '}}' }} interface {{ '{{' }} $labels.device {{ '}}' }} has encountered {{ '{{' }} printf \"%.0f\" $value {{ '}}' }} transmit errors in the last five minutes.\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
 
   - alert: HostNetworkInterfaceSaturated
-    expr: (rate(node_network_receive_bytes_total{device!~"^tap.*"}[1m]) + rate(node_network_transmit_bytes_total{device!~"^tap.*"}[1m])) / node_network_speed_bytes{device!~"^tap.*"} > 0.8
+    expr: (rate(node_network_receive_bytes_total{device!~"^tap.*"}[1m]) + rate(node_network_transmit_bytes_total{device!~"^tap.*"}[1m])) / node_network_speed_bytes{device!~"^tap.*"} > 0.95
     for: 1m
     labels:
       severity: warning
@@ -327,22 +327,22 @@ prometheus_server_rules_node:
       description: "S.M.A.R.T. attribute {{ '{{' }} $labels.name {{ '}}' }} for disk {{ '{{' }} $labels.device {{ '}}' }} on host {{ '{{' }} $labels.instance {{ '}}' }} has fallen below it's threshold.\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
 
   - alert: SmartmonReallocatedSectorsCount
-    expr: smartmon_attr_raw_value{name="reallocated_sector_ct"} > 0
+    expr: delta(smartmon_attr_raw_value{name="reallocated_sector_ct"}[24h]) > 0
     for: 0m
     labels:
       severity: warning
     annotations:
       summary: Host disk S.M.A.R.T. reports reallocated sectors (instance {{ '{{' }} $labels.instance {{ '}}' }})
-      description: "S.M.A.R.T. reports reallocated sectors for disk {{ '{{' }} $labels.device {{ '}}' }} on host {{ '{{' }} $labels.instance {{ '}}' }}, the drive might be failing.\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+      description: "S.M.A.R.T. reports multiple reallocated sectors within 24 hours for disk {{ '{{' }} $labels.device {{ '}}' }} on host {{ '{{' }} $labels.instance {{ '}}' }}, the drive might be failing.\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
 
   - alert: SmartmonReallocationEventCount
-    expr: smartmon_attr_raw_value{name="reallocated_event_count"} > 0
+    expr: delta(smartmon_attr_raw_value{name="reallocated_event_count"}[24h]) > 0
     for: 0m
     labels:
       severity: warning
     annotations:
       summary: Host disk S.M.A.R.T. reports reallocation events (instance {{ '{{' }} $labels.instance {{ '}}' }})
-      description: "S.M.A.R.T. reports reallocation events for disk {{ '{{' }} $labels.device {{ '}}' }} on host {{ '{{' }} $labels.instance {{ '}}' }}, the drive might be failing.\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+      description: "S.M.A.R.T. reports multiple reallocation events within 24 hours for disk {{ '{{' }} $labels.device {{ '}}' }} on host {{ '{{' }} $labels.instance {{ '}}' }}, the drive might be failing.\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
 
   - alert: SmartmonCurrentPendingSectors
     expr: smartmon_attr_raw_value{name="current_pending_sector"} > 0
-- 
cgit v1.2.3