prometheus: add ssl exporter

12 files changed, 239 insertions, 8 deletions

diff --git a/roles/monitoring/prometheus/exporter/TODO b/roles/monitoring/prometheus/exporter/TODO
index 57179464..53ded412 100644
--- a/roles/monitoring/prometheus/exporter/TODO
+++ b/roles/monitoring/prometheus/exporter/TODO
@@ -25,6 +25,3 @@ SNMP Exporter:
 Process Exporter:
   - https://github.com/ncabatoff/process-exporter
   - https://packages.debian.org/bullseye/prometheus-process-exporter
-
-SSL Exporter:
-  - https://github.com/ribbybibby/ssl_exporter
diff --git a/roles/monitoring/prometheus/exporter/meta/main.yml b/roles/monitoring/prometheus/exporter/meta/main.yml
index 68fce6cb..31dbd611 100644
--- a/roles/monitoring/prometheus/exporter/meta/main.yml
+++ b/roles/monitoring/prometheus/exporter/meta/main.yml
@@ -11,3 +11,5 @@ dependencies:
     when: "'node' in (prometheus_exporters_default | union(prometheus_exporters_extra))"
   - role: monitoring/prometheus/exporter/nut
     when: "'nut' in (prometheus_exporters_default | union(prometheus_exporters_extra))"
+  - role: monitoring/prometheus/exporter/ssl
+    when: "'ssl' in (prometheus_exporters_default | union(prometheus_exporters_extra))"
diff --git a/roles/monitoring/prometheus/exporter/ssl/defaults/main.yml b/roles/monitoring/prometheus/exporter/ssl/defaults/main.yml
new file mode 100644
index 00000000..d7edd3f4
--- /dev/null
+++ b/roles/monitoring/prometheus/exporter/ssl/defaults/main.yml
@@ -0,0 +1,16 @@
+---
+prometheus_exporter_ssl_modules:
+  tcp:
+    prober: tcp
+  http:
+    prober: https
+  https:
+    prober: https
+  file:
+    prober: file
+  kubernetes:
+    prober: kubernetes
+  kubeconfig:
+    prober: kubeconfig
+
+prometheus_exporter_ssl_modules_extra: {}
diff --git a/roles/monitoring/prometheus/exporter/ssl/handlers/main.yml b/roles/monitoring/prometheus/exporter/ssl/handlers/main.yml
new file mode 100644
index 00000000..2fb43f19
--- /dev/null
+++ b/roles/monitoring/prometheus/exporter/ssl/handlers/main.yml
@@ -0,0 +1,10 @@
+---
+- name: restart prometheus-ssl-exporter
+  service:
+    name: prometheus-ssl-exporter
+    state: restarted
+
+- name: reload nginx
+  service:
+    name: nginx
+    state: reloaded
diff --git a/roles/monitoring/prometheus/exporter/ssl/tasks/main.yml b/roles/monitoring/prometheus/exporter/ssl/tasks/main.yml
new file mode 100644
index 00000000..c57ea0b1
--- /dev/null
+++ b/roles/monitoring/prometheus/exporter/ssl/tasks/main.yml
@@ -0,0 +1,42 @@
+---
+  ## TODO: pin version
+- name: install apt packages
+  apt:
+    name: prom-exporter-ssl
+    state: present
+
+- name: create config directory
+  file:
+    path: /etc/prometheus/exporter/ssl
+    state: directory
+
+- name: generate configuration
+  template:
+    src: config.yml.j2
+    dest: /etc/prometheus/exporter/ssl/config.yml
+  notify: restart prometheus-ssl-exporter
+
+- name: generate systemd service unit
+  template:
+    src: service.j2
+    dest: /etc/systemd/system/prometheus-ssl-exporter.service
+  notify: restart prometheus-ssl-exporter
+
+- name: make sure prometheus-ssl-exporter is enabled and started
+  systemd:
+    name: prometheus-ssl-exporter.service
+    daemon_reload: yes
+    state: started
+    enabled: yes
+
+- name: register exporter
+  copy:
+    content: |
+      location = /ssl {
+          proxy_pass http://127.0.0.1:9219/metrics;
+      }
+      location = /ssl/probe {
+          proxy_pass http://127.0.0.1:9219/probe;
+      }
+    dest: /etc/prometheus/exporter/ssl.locations
+  notify: reload nginx
diff --git a/roles/monitoring/prometheus/exporter/ssl/templates/config.yml.j2 b/roles/monitoring/prometheus/exporter/ssl/templates/config.yml.j2
new file mode 100644
index 00000000..1ef84541
--- /dev/null
+++ b/roles/monitoring/prometheus/exporter/ssl/templates/config.yml.j2
@@ -0,0 +1,4 @@
+# {{ ansible_managed }}
+
+modules:
+  {{ prometheus_exporter_ssl_modules | combine(prometheus_exporter_ssl_modules_extra) | to_nice_yaml(indent=2) | indent(2) }}
diff --git a/roles/monitoring/prometheus/exporter/ssl/templates/service.j2 b/roles/monitoring/prometheus/exporter/ssl/templates/service.j2
new file mode 100644
index 00000000..fdd754a4
--- /dev/null
+++ b/roles/monitoring/prometheus/exporter/ssl/templates/service.j2
@@ -0,0 +1,30 @@
+[Unit]
+Description=Prometheus ssl exporter
+
+[Service]
+Restart=always
+ExecStart=/usr/bin/prometheus-ssl-exporter --web.listen-address="127.0.0.1:9219" --config.file=/etc/prometheus/exporter/ssl/config.yml
+ExecReload=/bin/kill -HUP $MAINPID
+
+# systemd hardening-options
+AmbientCapabilities=
+CapabilityBoundingSet=
+DeviceAllow=/dev/null rw
+DevicePolicy=strict
+LockPersonality=true
+MemoryDenyWriteExecute=true
+NoNewPrivileges=true
+PrivateDevices=true
+PrivateTmp=true
+ProtectControlGroups=true
+ProtectHome=true
+ProtectKernelModules=true
+ProtectKernelTunables=true
+ProtectSystem=strict
+RemoveIPC=true
+RestrictNamespaces=true
+RestrictRealtime=true
+SystemCallArchitectures=native
+
+[Install]
+WantedBy=multi-user.target
diff --git a/roles/monitoring/prometheus/server/defaults/main/main.yml b/roles/monitoring/prometheus/server/defaults/main/main.yml
index 1e0ccf78..3aea0509 100644
--- a/roles/monitoring/prometheus/server/defaults/main/main.yml
+++ b/roles/monitoring/prometheus/server/defaults/main/main.yml
@@ -20,6 +20,8 @@ prometheus_server_rules:
   blackbox/probe: "{{ prometheus_server_rules_blackbox__probe + prometheus_server_rules_blackbox__probe_extra }}"
   ipmi: "{{ prometheus_server_rules_ipmi + prometheus_server_rules_ipmi_extra }}"
   ipmi/remote: "{{ prometheus_server_rules_ipmi__remote + prometheus_server_rules_ipmi__remote_extra }}"
+  ssl: "{{ prometheus_server_rules_ssl + prometheus_server_rules_ssl_extra }}"
+  ssl/probe: "{{ prometheus_server_rules_ssl__probe + prometheus_server_rules_ssl__probe_extra }}"
 
 # prometheus_server_alertmanager:
 #   url: "127.0.0.1:9093"
diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_blackbox__probe.yml b/roles/monitoring/prometheus/server/defaults/main/rules_blackbox__probe.yml
index 9f9d2292..5a75abf0 100644
--- a/roles/monitoring/prometheus/server/defaults/main/rules_blackbox__probe.yml
+++ b/roles/monitoring/prometheus/server/defaults/main/rules_blackbox__probe.yml
@@ -1,4 +1,5 @@
 ---
+## https://awesome-prometheus-alerts.grep.to/rules#blackbox
 prometheus_server_rules_blackbox__probe_extra: []
 prometheus_server_rules_blackbox__probe:
   - alert: BlackboxProbeFailed
@@ -26,16 +27,16 @@ prometheus_server_rules_blackbox__probe:
       severity: warning
     annotations:
       summary: Blackbox SSL certificate will expire soon (instance {{ '{{' }} $labels.instance {{ '}}' }})
-      description: "SSL certificate expires in 30 days\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+      description: "SSL certificate expires in less than 30 days\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
 
-  - alert: BlackboxSslCertificateWillExpireSoon
-    expr: probe_ssl_earliest_cert_expiry - time() < 86400 * 3
+  - alert: BlackboxSslCertificateWillExpireVerySoon
+    expr: probe_ssl_earliest_cert_expiry - time() < 86400 * 7
     for: 0m
     labels:
       severity: critical
     annotations:
-      summary: Blackbox SSL certificate will expire soon (instance {{ '{{' }} $labels.instance {{ '}}' }})
-      description: "SSL certificate expires in 3 days\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+      summary: Blackbox SSL certificate will expire very soon (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "SSL certificate expires in less than 7 days\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
 
   - alert: BlackboxSslCertificateExpired
     expr: probe_ssl_earliest_cert_expiry - time() <= 0
diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_ssl.yml b/roles/monitoring/prometheus/server/defaults/main/rules_ssl.yml
new file mode 100644
index 00000000..06c7e027
--- /dev/null
+++ b/roles/monitoring/prometheus/server/defaults/main/rules_ssl.yml
@@ -0,0 +1,3 @@
+---
+prometheus_server_rules_ssl_extra: []
+prometheus_server_rules_ssl: []
diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_ssl__probe.yml b/roles/monitoring/prometheus/server/defaults/main/rules_ssl__probe.yml
new file mode 100644
index 00000000..8805de0a
--- /dev/null
+++ b/roles/monitoring/prometheus/server/defaults/main/rules_ssl__probe.yml
@@ -0,0 +1,119 @@
+---
+prometheus_server_rules_ssl__probe_extra: []
+prometheus_server_rules_ssl__probe:
+  - alert: SslCertificateProbeFailed
+    expr: ssl_probe_success == 0
+    for: 0m
+    labels:
+      severity: critical
+    annotations:
+      summary: SSL certificate probe failed (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "Failed to fetch SSL certificate information {{ '{{' }} $labels.instance {{ '}}' }}\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: SslCertificateExpiresSoon
+    expr: ssl_cert_not_after - time() < 86400 * 30
+    for: 0m
+    labels:
+      severity: warning
+    annotations:
+      summary: SSL certificate expires soon (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "The certificate {{ '{{' }} $labels.cn {{ '}}' }} is expiring in less than 30 days\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = BRACEOPEN $labels {{ '}}' }}"
+
+  - alert: SslCertificateExpiresVerySoon
+    expr: ssl_cert_not_after - time() < 86400 * 7
+    for: 0m
+    labels:
+      severity: critical
+    annotations:
+      summary: SSL certificate expires soon (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "The certificate {{ '{{' }} $labels.cn {{ '}}' }} is expiring in less than 7 days\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = BRACEOPEN $labels {{ '}}' }}"
+
+  - alert: SslCertificateChainExpiresSoon
+    expr: ssl_verified_cert_not_after - time() < 86400 * 30
+    for: 0m
+    labels:
+      severity: warning
+    annotations:
+      summary: SSL chain certificate expires soon (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "The chain certificate {{ '{{' }} $labels.cn {{ '}}' }} is expiring in less than 30 days\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = BRACEOPEN $labels {{ '}}' }}"
+
+  - alert: SslCertificateChainExpiresVerySoon
+    expr: ssl_verified_cert_not_after - time() < 86400 * 7
+    for: 0m
+    labels:
+      severity: critical
+    annotations:
+      summary: SSL chain certificate expires soon (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "The chain certificate {{ '{{' }} $labels.cn {{ '}}' }} is expiring in less than 7 days\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = BRACEOPEN $labels {{ '}}' }}"
+
+  - alert: SslCertificateOscpStatusUnknown
+    expr: ssl_ocsp_response_status == 2
+    for: 0m
+    labels:
+      severity: warning
+    annotations:
+      summary: SSL certificate OSCP status unknown (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "Failed to get the OSCP status for {{ '{{' }} $labels.cn {{ '}}' }}\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: SslCertificateOscpStatusRevoked
+    expr: ssl_ocsp_response_status == 1
+    for: 0m
+    labels:
+      severity: critical
+    annotations:
+      summary: SSL certificate revoked (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "OCSP reports SSL certificate {{ '{{' }} $labels.cn {{ '}}' }} as revoked\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: SslCertificateFileExpiresSoon
+    expr: ssl_file_cert_not_after - time() < 86400 * 30
+    for: 0m
+    labels:
+      severity: warning
+    annotations:
+      summary: SSL certificate expires soon (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "The certificate in {{ '{{' }} $labels.file {{ '}}' }} is expiring in less than 30 days\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = BRACEOPEN $labels {{ '}}' }}"
+
+  - alert: SslCertificateFileExpiresVerySoon
+    expr: ssl_file_cert_not_after - time() < 86400 * 7
+    for: 0m
+    labels:
+      severity: critical
+    annotations:
+      summary: SSL certificate expires soon (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "The certificate in {{ '{{' }} $labels.file {{ '}}' }} is expiring in less than 7 days\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = BRACEOPEN $labels {{ '}}' }}"
+
+  - alert: SslCertificateKubernetesExpiresSoon
+    expr: ssl_kubernetes_cert_not_after - time() < 86400 * 30
+    for: 0m
+    labels:
+      severity: warning
+    annotations:
+      summary: SSL kubernetes certificate expires soon (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "The certificate in kubernetes secret {{ '{{' }} $labels.namespace {{ '}}' }}/{{ '{{' }} $labels.secret {{ '}}' }} is expiring in less than 30 days\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = BRACEOPEN $labels {{ '}}' }}"
+
+  - alert: SslCertificateKubernetesExpiresVerySoon
+    expr: ssl_kubernetes_cert_not_after - time() < 86400 * 7
+    for: 0m
+    labels:
+      severity: critical
+    annotations:
+      summary: SSL kubernetes certificate expires soon (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "The certificate in kubernetes secret {{ '{{' }} $labels.namespace {{ '}}' }}/{{ '{{' }} $labels.secret {{ '}}' }} is expiring in less than 7 days\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = BRACEOPEN $labels {{ '}}' }}"
+
+  - alert: SslCertificateKubeconfigExpiresSoon
+    expr: ssl_kubeconfig_cert_not_after - time() < 86400 * 30
+    for: 0m
+    labels:
+      severity: warning
+    annotations:
+      summary: SSL kubeconfig certificate expires soon (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "The certificate in kubeconfig {{ '{{' }} $labels.kubeconfig {{ '}}' }} is expiring in less than 30 days\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = BRACEOPEN $labels {{ '}}' }}"
+
+  - alert: SslCertificateKubeconfigExpiresVerySoon
+    expr: ssl_kubeconfig_cert_not_after - time() < 86400 * 7
+    for: 0m
+    labels:
+      severity: critical
+    annotations:
+      summary: SSL kubeconfig certificate expires soon (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "The certificate in kubeconfig {{ '{{' }} $labels.kubeconfig {{ '}}' }} is expiring in less than 7 days\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = BRACEOPEN $labels {{ '}}' }}"
diff --git a/roles/monitoring/prometheus/server/templates/targets/ssl/probe.yml.j2 b/roles/monitoring/prometheus/server/templates/targets/ssl/probe.yml.j2
new file mode 100644
index 00000000..4e336873
--- /dev/null
+++ b/roles/monitoring/prometheus/server/templates/targets/ssl/probe.yml.j2
@@ -0,0 +1,5 @@
+- targets: [ '{{ hostvars[target.exporter_hostname].prometheus_scrape_endpoint }}' ]
+  labels:
+    instance: '{{ target.instance }}'
+    __param_target: '{{ target.config.target }}'
+    __param_module: '{{ target.config.module }}'