diff options
| author | Christian Pointner <equinox@spreadspace.org> | 2023-09-21 18:37:34 +0200 |
|---|---|---|
| committer | Christian Pointner <equinox@spreadspace.org> | 2023-09-21 18:37:34 +0200 |
| commit | 67dbf646f686dac08419cad521a178a793be2c22 (patch) | |
| tree | 8bacf331cb0fe01656427a3428f0ed3f148d3562 /roles/monitoring | |
| parent | uacme: eab hmac key is most likely already encoded and doesn't need additiona... (diff) | |
fix smartctl text collector and add some alerts for nvme metrics
Diffstat (limited to 'roles/monitoring')
| -rw-r--r-- | roles/monitoring/prometheus/exporter/node/templates/textfile-collector-scripts/smartmon.j2 | 1 | ||||
| -rw-r--r-- | roles/monitoring/prometheus/server/defaults/main/rules_node.yml | 18 |
2 files changed, 19 insertions, 0 deletions
diff --git a/roles/monitoring/prometheus/exporter/node/templates/textfile-collector-scripts/smartmon.j2 b/roles/monitoring/prometheus/exporter/node/templates/textfile-collector-scripts/smartmon.j2 index c6bfba28..f564cc22 100644 --- a/roles/monitoring/prometheus/exporter/node/templates/textfile-collector-scripts/smartmon.j2 +++ b/roles/monitoring/prometheus/exporter/node/templates/textfile-collector-scripts/smartmon.j2 @@ -491,6 +491,7 @@ def collect_nvme_metrics(device): attribute_lines = attributes.strip().split('\n')[6:] for line in attribute_lines: label, value = line.split(':') + value = value.replace(',', '') if label == 'Available Spare': metrics['available_spare_ratio'].labels( device.base_labels["device"], device.base_labels["disk"] diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_node.yml b/roles/monitoring/prometheus/server/defaults/main/rules_node.yml index 5493da57..fe6e781b 100644 --- a/roles/monitoring/prometheus/server/defaults/main/rules_node.yml +++ b/roles/monitoring/prometheus/server/defaults/main/rules_node.yml @@ -393,6 +393,24 @@ prometheus_server_rules_node: summary: Host disk S.M.A.R.T. reports offline uncorrectable errors (instance {{ '{{' }} $labels.instance {{ '}}' }}) description: "S.M.A.R.T. reports offline uncorrectable errors for disk {{ '{{' }} $labels.device {{ '}}' }} on host {{ '{{' }} $labels.instance {{ '}}' }}, the drive might be failing.\n VALUE = {{ '{{' }} $value {{ '}}' }}\n LABELS = {{ '{{' }} $labels {{ '}}' }}" + - alert: SmartmonNVMEMediaErrors + expr: delta(smartmon_media_errors_total[72h]) > 0 + for: 0m + labels: + severity: warning + annotations: + summary: Host disk S.M.A.R.T. reports NVMe media errors. (instance {{ '{{' }} $labels.instance {{ '}}' }}) + description: "S.M.A.R.T. reports NVMe media errors for disk {{ '{{' }} $labels.device {{ '}}' }} on host {{ '{{' }} $labels.instance {{ '}}' }}, the drive might be failing.\n VALUE = {{ '{{' }} $value {{ '}}' }}\n LABELS = {{ '{{' }} $labels {{ '}}' }}" + + - alert: SmartmonNVMEPercentageUsed + expr: smartmon_percentage_used_ratio > 80 + for: 0m + labels: + severity: warning + annotations: + summary: Host disk S.M.A.R.T. reports worn out disk. (instance {{ '{{' }} $labels.instance {{ '}}' }}) + description: "S.M.A.R.T. reports NVMe high usage of device {{ '{{' }} $labels.device {{ '}}' }} on host {{ '{{' }} $labels.instance {{ '}}' }}, the drive might be completely worn out soon.\n VALUE = {{ '{{' }} $value {{ '}}' }}\n LABELS = {{ '{{' }} $labels {{ '}}' }}" + ## syncoid-pull scripts - alert: SyncoidPullJobTooLongAgo expr: time() - syncoid_pull_run > (24 * 3600) |