fix and finalize ipmi exporter

4 files changed, 144 insertions, 7 deletions

diff --git a/roles/monitoring/prometheus/exporter/base/tasks/main.yml b/roles/monitoring/prometheus/exporter/base/tasks/main.yml
index 5f42867d..c69c6e05 100644
--- a/roles/monitoring/prometheus/exporter/base/tasks/main.yml
+++ b/roles/monitoring/prometheus/exporter/base/tasks/main.yml
@@ -13,6 +13,11 @@
     home: /nonexistent
     create_home: no
 
+- name: create base directory for exporter configs
+  file:
+    path: /etc/prometheus/exporter
+    state: directory
+
 - name: create TLS certificate and key
   import_tasks: tls.yml
 
diff --git a/roles/monitoring/prometheus/exporter/base/tasks/tls.yml b/roles/monitoring/prometheus/exporter/base/tasks/tls.yml
index e34025e4..083ca930 100644
--- a/roles/monitoring/prometheus/exporter/base/tasks/tls.yml
+++ b/roles/monitoring/prometheus/exporter/base/tasks/tls.yml
@@ -25,7 +25,7 @@
     owner: prometheus-exporter
     group: prometheus-exporter
     mode: 0400
-  notify: restart prometheus-exporter-exporter
+  notify: reload nginx
 
 - name: create signing request for exporter certificate
   openssl_csr:
@@ -86,7 +86,7 @@
   copy:
     content: "{{ prometheus_exporter_server_cert.certificate }}"
     dest: /etc/ssl/prometheus/exporter/crt.pem
-  notify: restart prometheus-exporter-exporter
+  notify: reload nginx
 
 - name: slurp CA certificate
   delegate_to: "{{ prometheus_server }}"
diff --git a/roles/monitoring/prometheus/exporter/ipmi/templates/service.j2 b/roles/monitoring/prometheus/exporter/ipmi/templates/service.j2
index 465215e8..d862e299 100644
--- a/roles/monitoring/prometheus/exporter/ipmi/templates/service.j2
+++ b/roles/monitoring/prometheus/exporter/ipmi/templates/service.j2
@@ -1,22 +1,21 @@
 [Unit]
 Description=Prometheus ipmi exporter
+After=systemd-modules-load.service
 
 [Service]
 Restart=always
-User=prometheus-exporter
 ExecStart=/usr/bin/prometheus-ipmi-exporter --web.listen-address="127.0.0.1:9290" --config.file=/etc/prometheus/exporter/ipmi/config.yml --freeipmi.path="/usr/sbin"
 ExecReload=/bin/kill -HUP $MAINPID
 
-{# TODO: test which hardening options need to be removed for IPMI to work... #}
 # systemd hardening-options
 AmbientCapabilities=
 CapabilityBoundingSet=
 DeviceAllow=/dev/null rw
+DeviceAllow=char-ipmidev rw
 DevicePolicy=strict
 LockPersonality=true
 MemoryDenyWriteExecute=true
 NoNewPrivileges=true
-PrivateDevices=true
 PrivateTmp=true
 ProtectControlGroups=true
 ProtectHome=true
diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_ipmi.yml b/roles/monitoring/prometheus/server/defaults/main/rules_ipmi.yml
index 41dcd7e9..31c15d51 100644
--- a/roles/monitoring/prometheus/server/defaults/main/rules_ipmi.yml
+++ b/roles/monitoring/prometheus/server/defaults/main/rules_ipmi.yml
@@ -1,4 +1,137 @@
 ---
 prometheus_server_rules_ipmi_extra: []
-prometheus_server_rules_ipmi: []
-## TODO: add common IPMI alert rules
+prometheus_server_rules_ipmi:
+  - alert: IpmiFailedToScrapeCollector
+    expr: ipmi_up == 0
+    for: 2m
+    labels:
+      severity: warning
+    annotations:
+      summary: IPMI collector failed to scrape (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "The IPMI collector {{ '{{' }} $labels.collector {{ '}}' }} could not be scraped.\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: IpmiSensorStateWarning
+    expr: ipmi_sensor_state == 1
+    for: 2m
+    labels:
+      severity: warning
+    annotations:
+      summary: IPMI Sensor state is warning (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "The IPMI sensor {{ '{{' }} $labels.name {{ '}}' }} has state: warning.\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: IpmiSensorStateCritical
+    expr: ipmi_sensor_state == 2
+    for: 2m
+    labels:
+      severity: critical
+    annotations:
+      summary: IPMI Sensor state is critical (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "The IPMI sensor {{ '{{' }} $labels.name {{ '}}' }} has state: critical.\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: IpmiFanSpeedStateWarning
+    expr: ipmi_fan_speed_state == 1
+    for: 2m
+    labels:
+      severity: warning
+    annotations:
+      summary: IPMI fan-speed state is warning (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "The IPMI fanspeed {{ '{{' }} $labels.name {{ '}}' }} has state: warning.\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: IpmiFanSpeedStateCritical
+    expr: ipmi_fan_speed_state == 2
+    for: 2m
+    labels:
+      severity: critical
+    annotations:
+      summary: IPMI fan-speed state is critical (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "The IPMI fan speed {{ '{{' }} $labels.name {{ '}}' }} has state: critical.\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: IpmiTemperatureStateWarning
+    expr: ipmi_temperature_state == 1
+    for: 2m
+    labels:
+      severity: warning
+    annotations:
+      summary: IPMI Temperature state is warning (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "The IPMI temperature {{ '{{' }} $labels.name {{ '}}' }} has state: warning.\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: IpmiTemperatureStateCritical
+    expr: ipmi_temperature_state == 2
+    for: 2m
+    labels:
+      severity: critical
+    annotations:
+      summary: IPMI Temperature state is critical (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "The IPMI temperature {{ '{{' }} $labels.name {{ '}}' }} has state: critical.\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: IpmiVoltageStateWarning
+    expr: ipmi_voltage_state == 1
+    for: 2m
+    labels:
+      severity: warning
+    annotations:
+      summary: IPMI Voltage state is warning (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "The IPMI voltage {{ '{{' }} $labels.name {{ '}}' }} has state: warning.\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: IpmiVoltageStateCritical
+    expr: ipmi_voltage_state == 2
+    for: 2m
+    labels:
+      severity: critical
+    annotations:
+      summary: IPMI Voltage state is critical (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "The IPMI voltage {{ '{{' }} $labels.name {{ '}}' }} has state: critical.\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: IpmiCurrentStateWarning
+    expr: ipmi_current_state == 1
+    for: 2m
+    labels:
+      severity: warning
+    annotations:
+      summary: IPMI Current state is warning (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "The IPMI current {{ '{{' }} $labels.name {{ '}}' }} has state: warning.\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: IpmiCurrentStateCritical
+    expr: ipmi_current_state == 2
+    for: 2m
+    labels:
+      severity: critical
+    annotations:
+      summary: IPMI Current state is critical (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "The IPMI current {{ '{{' }} $labels.name {{ '}}' }} has state: critical.\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: IpmiPowerStateWarning
+    expr: ipmi_power_state == 1
+    for: 2m
+    labels:
+      severity: warning
+    annotations:
+      summary: IPMI Power state is warning (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "The IPMI power {{ '{{' }} $labels.name {{ '}}' }} has state: warning.\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: IpmiPowerStateCritical
+    expr: ipmi_power_state == 2
+    for: 2m
+    labels:
+      severity: critical
+    annotations:
+      summary: IPMI Power state is critical (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "The IPMI power {{ '{{' }} $labels.name {{ '}}' }} has state: critical.\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: IpmiSystemEventLogManyEvents
+    expr: rate(ipmi_sel_logs_count[2m]) > 0.1
+    for: 1m
+    labels:
+      severity: warning
+    annotations:
+      summary: IPMI SEL grows quickly (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "The IPMI system log contains too many new events.\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: IpmiSystemEventLogFull
+    expr: ipmi_sel_free_space_bytes < 100
+    for: 0m
+    labels:
+      severity: warning
+    annotations:
+      summary: IPMI SEL is full (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "The IPMI system log ran out of space.\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"