prometheus: add blackbox exporter

9 files changed, 168 insertions, 2 deletions

diff --git a/roles/monitoring/prometheus/exporter/blackbox/defaults/main.yml b/roles/monitoring/prometheus/exporter/blackbox/defaults/main.yml
new file mode 100644
index 00000000..fcf66555
--- /dev/null
+++ b/roles/monitoring/prometheus/exporter/blackbox/defaults/main.yml
@@ -0,0 +1,22 @@
+---
+prometheus_exporter_blackbox_modules:
+  icmp:
+    prober: icmp
+  tcp_connect:
+    prober: tcp
+  tcp_tls:
+    prober: tcp
+    tcp:
+      tls: true
+      tls_config:
+        insecure_skip_verify: true
+  http_2xx:
+    prober: http
+  ssh_banner:
+    prober: tcp
+    tcp:
+      query_response:
+      - expect: "^SSH-2.0-"
+      - send: "SSH-2.0-blackbox-ssh-check"
+
+prometheus_exporter_blackbox_modules_extra: {}
diff --git a/roles/monitoring/prometheus/exporter/blackbox/handlers/main.yml b/roles/monitoring/prometheus/exporter/blackbox/handlers/main.yml
new file mode 100644
index 00000000..99a416e2
--- /dev/null
+++ b/roles/monitoring/prometheus/exporter/blackbox/handlers/main.yml
@@ -0,0 +1,16 @@
+---
+- name: restart prometheus-blackbox-exporter
+  service:
+    name: prometheus-blackbox-exporter
+    state: restarted
+
+- name: reload prometheus-blackbox-exporter
+  service:
+    name: prometheus-blackbox-exporter
+    state: reloaded
+
+- name: reload prometheus-exporter-exporter
+  service:
+    name: prometheus-exporter-exporter
+    ## TODO: implement reload once exporter_exporter supports this...
+    state: restarted
diff --git a/roles/monitoring/prometheus/exporter/blackbox/tasks/main.yml b/roles/monitoring/prometheus/exporter/blackbox/tasks/main.yml
new file mode 100644
index 00000000..3b8e997d
--- /dev/null
+++ b/roles/monitoring/prometheus/exporter/blackbox/tasks/main.yml
@@ -0,0 +1,39 @@
+---
+- name: install apt packages
+  apt:
+    name: prom-exporter-blackbox
+    state: present
+
+- name: create config directory
+  file:
+    path: /etc/prometheus/exporter/blackbox
+    state: directory
+
+- name: generate configuration
+  template:
+    src: config.yml.j2
+    dest: /etc/prometheus/exporter/blackbox/config.yml
+  notify: reload prometheus-blackbox-exporter
+
+- name: generate systemd service unit
+  template:
+    src: service.j2
+    dest: /etc/systemd/system/prometheus-blackbox-exporter.service
+  notify: restart prometheus-blackbox-exporter
+
+- name: make sure prometheus-exporter-exporter is enabled and started
+  systemd:
+    name: prometheus-blackbox-exporter.service
+    daemon_reload: yes
+    state: started
+    enabled: yes
+
+- name: register exporter
+  copy:
+    content: |
+      method: http
+      http:
+        port: 9115
+        path: '/probe'
+    dest: /etc/prometheus/exporter/enabled/blackbox.yml
+  notify: reload prometheus-exporter-exporter
diff --git a/roles/monitoring/prometheus/exporter/blackbox/templates/config.yml.j2 b/roles/monitoring/prometheus/exporter/blackbox/templates/config.yml.j2
new file mode 100644
index 00000000..01e3f7a0
--- /dev/null
+++ b/roles/monitoring/prometheus/exporter/blackbox/templates/config.yml.j2
@@ -0,0 +1,4 @@
+# {{ ansible_managed }}
+
+modules:
+  {{ prometheus_exporter_blackbox_modules | combine(prometheus_exporter_blackbox_modules_extra) | to_nice_yaml(indent=2) | indent(2)}}
diff --git a/roles/monitoring/prometheus/exporter/blackbox/templates/service.j2 b/roles/monitoring/prometheus/exporter/blackbox/templates/service.j2
new file mode 100644
index 00000000..c9c5712c
--- /dev/null
+++ b/roles/monitoring/prometheus/exporter/blackbox/templates/service.j2
@@ -0,0 +1,32 @@
+[Unit]
+Description=Prometheus blackbox exporter
+
+[Service]
+Restart=always
+User=prometheus-exporter
+ExecStart=/usr/bin/prometheus-blackbox-exporter --web.listen-address="127.0.0.1:9115" --config.file=/etc/prometheus/exporter/blackbox/config.yml
+ExecReload=/bin/kill -HUP $MAINPID
+
+# systemd hardening-options
+AmbientCapabilities=CAP_NET_RAW
+CapabilityBoundingSet=CAP_NET_RAW
+DeviceAllow=/dev/null rw
+DevicePolicy=strict
+LockPersonality=true
+MemoryDenyWriteExecute=true
+NoNewPrivileges=true
+PrivateDevices=true
+PrivateTmp=true
+PrivateUsers=true
+ProtectControlGroups=true
+ProtectHome=true
+ProtectKernelModules=true
+ProtectKernelTunables=true
+ProtectSystem=strict
+RemoveIPC=true
+RestrictNamespaces=true
+RestrictRealtime=true
+SystemCallArchitectures=native
+
+[Install]
+WantedBy=multi-user.target
diff --git a/roles/monitoring/prometheus/exporter/node/templates/service.j2 b/roles/monitoring/prometheus/exporter/node/templates/service.j2
index 801850ed..7aa2834a 100644
--- a/roles/monitoring/prometheus/exporter/node/templates/service.j2
+++ b/roles/monitoring/prometheus/exporter/node/templates/service.j2
@@ -1,5 +1,5 @@
 [Unit]
-Description=Prometheus exporter
+Description=Prometheus node exporter
 
 [Service]
 Restart=always
diff --git a/roles/monitoring/prometheus/server/handlers/main.yml b/roles/monitoring/prometheus/server/handlers/main.yml
index edeba752..bf8735e9 100644
--- a/roles/monitoring/prometheus/server/handlers/main.yml
+++ b/roles/monitoring/prometheus/server/handlers/main.yml
@@ -3,3 +3,8 @@
   service:
     name: prometheus
     state: restarted
+
+- name: reload prometheus
+  service:
+    name: prometheus
+    state: reloaded
diff --git a/roles/monitoring/prometheus/server/tasks/main.yml b/roles/monitoring/prometheus/server/tasks/main.yml
index 61660a03..6b030fb4 100644
--- a/roles/monitoring/prometheus/server/tasks/main.yml
+++ b/roles/monitoring/prometheus/server/tasks/main.yml
@@ -54,7 +54,7 @@
   template:
     src: prometheus.yml.j2
     dest: /etc/prometheus/prometheus.yml
-  notify: restart prometheus
+  notify: reload prometheus
 
 - name: generate systemd service unit
   template:
diff --git a/roles/monitoring/prometheus/server/templates/prometheus.yml.j2 b/roles/monitoring/prometheus/server/templates/prometheus.yml.j2
index e94ea043..eb77d6d1 100644
--- a/roles/monitoring/prometheus/server/templates/prometheus.yml.j2
+++ b/roles/monitoring/prometheus/server/templates/prometheus.yml.j2
@@ -1,3 +1,5 @@
+# {{ ansible_managed }}
+
 global:
   scrape_interval:     15s
   evaluation_interval: 15s
@@ -27,3 +29,49 @@ scrape_configs:
     - files:
       - "/etc/prometheus/jobs/{{ job }}/*.yml"
 {% endfor %}
+
+  ## TODO: temporary test
+  - job_name: 'ping'
+    metrics_path: /proxy
+    params:
+      module:
+      - blackbox
+      - icmp
+    scheme: https
+    tls_config:
+      ca_file: /etc/ssl/prometheus/ca-crt.pem
+      cert_file: /etc/ssl/prometheus/server/exporter-crt.pem
+      key_file: /etc/ssl/prometheus/server/exporter-key.pem
+    static_configs:
+      - targets:
+        - 62.99.185.129
+        - 9.9.9.9
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: 192.168.32.230:9999
+
+  - job_name: 'tcp_tls'
+    metrics_path: /proxy
+    params:
+      module:
+      - blackbox
+      - tcp_tls
+    scheme: https
+    tls_config:
+      ca_file: /etc/ssl/prometheus/ca-crt.pem
+      cert_file: /etc/ssl/prometheus/server/exporter-crt.pem
+      key_file: /etc/ssl/prometheus/server/exporter-key.pem
+    static_configs:
+      - targets:
+        - web.chaos-at-home.org:443
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: 192.168.32.230:9999