From b362c4a1b1462b1ccf26750752c2147615508f52 Mon Sep 17 00:00:00 2001
From: Christian Pointner <equinox@spreadspace.org>
Date: Sat, 17 Dec 2022 18:21:12 +0100
Subject: add prometheus snmp exporter

---
 roles/monitoring/prometheus/exporter/meta/main.yml |  2 +
 .../prometheus/exporter/snmp/defaults/main.yml     |  8 +++
 .../prometheus/exporter/snmp/handlers/main.yml     | 15 +++++
 .../prometheus/exporter/snmp/tasks/main.yml        | 64 ++++++++++++++++++++++
 .../exporter/snmp/templates/config.yml.j2          |  3 +
 .../prometheus/exporter/snmp/templates/service.j2  | 31 +++++++++++
 .../prometheus/server/defaults/main/main.yml       |  2 +
 .../prometheus/server/defaults/main/rules_snmp.yml | 11 ++++
 .../server/defaults/main/rules_snmp__probe.yml     | 11 ++++
 .../server/templates/targets/snmp/probe.yml.j2     |  5 ++
 10 files changed, 152 insertions(+)
 create mode 100644 roles/monitoring/prometheus/exporter/snmp/defaults/main.yml
 create mode 100644 roles/monitoring/prometheus/exporter/snmp/handlers/main.yml
 create mode 100644 roles/monitoring/prometheus/exporter/snmp/tasks/main.yml
 create mode 100644 roles/monitoring/prometheus/exporter/snmp/templates/config.yml.j2
 create mode 100644 roles/monitoring/prometheus/exporter/snmp/templates/service.j2
 create mode 100644 roles/monitoring/prometheus/server/defaults/main/rules_snmp.yml
 create mode 100644 roles/monitoring/prometheus/server/defaults/main/rules_snmp__probe.yml
 create mode 100644 roles/monitoring/prometheus/server/templates/targets/snmp/probe.yml.j2

(limited to 'roles/monitoring')

diff --git a/roles/monitoring/prometheus/exporter/meta/main.yml b/roles/monitoring/prometheus/exporter/meta/main.yml
index 3999b608..21b43396 100644
--- a/roles/monitoring/prometheus/exporter/meta/main.yml
+++ b/roles/monitoring/prometheus/exporter/meta/main.yml
@@ -17,3 +17,5 @@ dependencies:
     when: "'smokeping' in (prometheus_exporters_default | union(prometheus_exporters_extra))"
   - role: monitoring/prometheus/exporter/bind
     when: "'bind' in (prometheus_exporters_default | union(prometheus_exporters_extra))"
+  - role: monitoring/prometheus/exporter/snmp
+    when: "'snmp' in (prometheus_exporters_default | union(prometheus_exporters_extra))"
diff --git a/roles/monitoring/prometheus/exporter/snmp/defaults/main.yml b/roles/monitoring/prometheus/exporter/snmp/defaults/main.yml
new file mode 100644
index 00000000..a8c67a23
--- /dev/null
+++ b/roles/monitoring/prometheus/exporter/snmp/defaults/main.yml
@@ -0,0 +1,8 @@
+---
+# prometheus_exporter_snmp_version:
+
+## the default config will be replaced with this
+# prometheus_exporter_snmp_config:
+
+## this extends the default config
+# prometheus_exporter_snmp_config_extra:
diff --git a/roles/monitoring/prometheus/exporter/snmp/handlers/main.yml b/roles/monitoring/prometheus/exporter/snmp/handlers/main.yml
new file mode 100644
index 00000000..f6059aa6
--- /dev/null
+++ b/roles/monitoring/prometheus/exporter/snmp/handlers/main.yml
@@ -0,0 +1,15 @@
+---
+- name: restart prometheus-snmp-exporter
+  service:
+    name: prometheus-snmp-exporter
+    state: restarted
+
+- name: reload prometheus-snmp-exporter
+  service:
+    name: prometheus-snmp-exporter
+    state: reloaded
+
+- name: reload nginx
+  service:
+    name: nginx
+    state: reloaded
diff --git a/roles/monitoring/prometheus/exporter/snmp/tasks/main.yml b/roles/monitoring/prometheus/exporter/snmp/tasks/main.yml
new file mode 100644
index 00000000..8fc93f0b
--- /dev/null
+++ b/roles/monitoring/prometheus/exporter/snmp/tasks/main.yml
@@ -0,0 +1,64 @@
+---
+- name: generate apt pin file for exporter-snmp package
+  when: prometheus_exporter_snmp_version is defined
+  copy:
+    dest: "/etc/apt/preferences.d/prom-exporter-snmp.pref"
+    content: |
+      Package: prom-exporter-snmp
+      Pin: version {{ prometheus_exporter_snmp_version }}-1
+      Pin-Priority: 1001
+
+- name: remove apt pin file for exporter-snmp package
+  when: prometheus_exporter_snmp_version is not defined
+  file:
+    path: "/etc/apt/preferences.d/prom-exporter-snmp.pref"
+    state: absent
+
+- name: install apt packages
+  apt:
+    name: "prom-exporter-snmp{% if prometheus_exporter_snmp_version is defined %}={{ prometheus_exporter_snmp_version }}-1{% endif %}"
+    state: present
+    allow_downgrade: yes
+  notify: restart prometheus-snmp-exporter
+
+- name: generate configuration
+  when: prometheus_exporter_snmp_config is defined
+  template:
+    src: config.yml.j2
+    dest: /etc/prometheus/exporter/snmp/config.yml
+  notify: reload prometheus-snmp-exporter
+
+- name: extend the default configuration
+  when:
+  - prometheus_exporter_snmp_config is not defined
+  - prometheus_exporter_snmp_config_extra is defined
+  blockinfile:
+    path: /etc/prometheus/exporter/snmp/config.yml
+    block: |
+      {{ prometheus_exporter_snmp_config_extra | to_nice_yaml(indent=2) }}
+  notify: reload prometheus-snmp-exporter
+
+- name: generate systemd service unit
+  template:
+    src: service.j2
+    dest: /etc/systemd/system/prometheus-snmp-exporter.service
+  notify: restart prometheus-snmp-exporter
+
+- name: make sure prometheus-snmp-exporter is enabled and started
+  systemd:
+    name: prometheus-snmp-exporter.service
+    daemon_reload: yes
+    state: started
+    enabled: yes
+
+- name: register exporter
+  copy:
+    content: |
+      location = /snmp {
+          proxy_pass http://127.0.0.1:9116/metrics;
+      }
+      location = /snmp/probe {
+          proxy_pass http://127.0.0.1:9116/snmp;
+      }
+    dest: /etc/prometheus/exporter/snmp.locations
+  notify: reload nginx
diff --git a/roles/monitoring/prometheus/exporter/snmp/templates/config.yml.j2 b/roles/monitoring/prometheus/exporter/snmp/templates/config.yml.j2
new file mode 100644
index 00000000..c75b8469
--- /dev/null
+++ b/roles/monitoring/prometheus/exporter/snmp/templates/config.yml.j2
@@ -0,0 +1,3 @@
+# {{ ansible_managed }}
+
+{{ prometheus_exporter_snmp_config | combine(prometheus_exporter_snmp_config_extra | default({})) | to_nice_yaml(indent=2) }}
diff --git a/roles/monitoring/prometheus/exporter/snmp/templates/service.j2 b/roles/monitoring/prometheus/exporter/snmp/templates/service.j2
new file mode 100644
index 00000000..7fccb2da
--- /dev/null
+++ b/roles/monitoring/prometheus/exporter/snmp/templates/service.j2
@@ -0,0 +1,31 @@
+[Unit]
+Description=Prometheus snmp exporter
+
+[Service]
+Restart=always
+User=prometheus-exporter
+ExecStart=/usr/bin/prometheus-snmp-exporter --web.listen-address="127.0.0.1:9116" --config.file=/etc/prometheus/exporter/snmp/config.yml
+ExecReload=/bin/kill -HUP $MAINPID
+
+# systemd hardening-options
+AmbientCapabilities=
+CapabilityBoundingSet=
+DeviceAllow=/dev/null rw
+DevicePolicy=strict
+LockPersonality=true
+MemoryDenyWriteExecute=true
+NoNewPrivileges=true
+PrivateDevices=true
+PrivateTmp=true
+ProtectControlGroups=true
+ProtectHome=true
+ProtectKernelModules=true
+ProtectKernelTunables=true
+ProtectSystem=strict
+RemoveIPC=true
+RestrictNamespaces=true
+RestrictRealtime=true
+SystemCallArchitectures=native
+
+[Install]
+WantedBy=multi-user.target
diff --git a/roles/monitoring/prometheus/server/defaults/main/main.yml b/roles/monitoring/prometheus/server/defaults/main/main.yml
index 375b3458..b377a7f0 100644
--- a/roles/monitoring/prometheus/server/defaults/main/main.yml
+++ b/roles/monitoring/prometheus/server/defaults/main/main.yml
@@ -29,6 +29,8 @@ prometheus_server_rules:
   ssl/probe: "{{ prometheus_server_rules_ssl__probe + prometheus_server_rules_ssl__probe_extra }}"
   smokeping: "{{ prometheus_server_rules_smokeping + prometheus_server_rules_smokeping_extra }}"
   bind: "{{ prometheus_server_rules_bind + prometheus_server_rules_bind_extra }}"
+  snmp: "{{ prometheus_server_rules_snmp + prometheus_server_rules_snmp_extra }}"
+  snmp/probe: "{{ prometheus_server_rules_snmp__probe + prometheus_server_rules_snmp__probe_extra }}"
 
 # prometheus_server_alertmanager:
 #   url: "127.0.0.1:9093"
diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_snmp.yml b/roles/monitoring/prometheus/server/defaults/main/rules_snmp.yml
new file mode 100644
index 00000000..7aec66c4
--- /dev/null
+++ b/roles/monitoring/prometheus/server/defaults/main/rules_snmp.yml
@@ -0,0 +1,11 @@
+---
+prometheus_server_rules_snmp_extra: []
+prometheus_server_rules_snmp:
+  - alert: SNMPRequestErrors
+    expr: increase(snmp_request_errors_total[1m]) > 0
+    for: 5m
+    labels:
+      severity: warning
+    annotations:
+      summary: SNMP Exporter reports request errors (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "SNMP expoter on {{ '{{' }} $labels.instance {{ '}}' }} reports request errors \n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_snmp__probe.yml b/roles/monitoring/prometheus/server/defaults/main/rules_snmp__probe.yml
new file mode 100644
index 00000000..548144c2
--- /dev/null
+++ b/roles/monitoring/prometheus/server/defaults/main/rules_snmp__probe.yml
@@ -0,0 +1,11 @@
+---
+prometheus_server_rules_snmp__probe_extra: []
+prometheus_server_rules_snmp__probe:
+  - alert: SNMPSlowScrape
+    expr: snmp_scrape_duration_seconds > 5
+    for: 0m
+    labels:
+      severity: warning
+    annotations:
+      summary: SNMP slow scrape  (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "Scraping SNMP target took more than 5 seconds \n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
diff --git a/roles/monitoring/prometheus/server/templates/targets/snmp/probe.yml.j2 b/roles/monitoring/prometheus/server/templates/targets/snmp/probe.yml.j2
new file mode 100644
index 00000000..47e6dceb
--- /dev/null
+++ b/roles/monitoring/prometheus/server/templates/targets/snmp/probe.yml.j2
@@ -0,0 +1,5 @@
+- targets: [ '{{ (target.exporter_hostname == prometheus_server) | ternary('127.0.0.1:9999', hostvars[target.exporter_hostname].prometheus_scrape_endpoint) }}' ]
+  labels:
+    instance: '{{ target.instance }}'
+    __param_target: '{{ target.config.target }}'
+    __param_module: '{{ target.config.module }}'
-- 
cgit v1.2.3