From c49412b8f31f551ec90b3dcd1d8e8e867a2b1680 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Tue, 28 Sep 2021 18:02:27 +0200 Subject: prometheus: add ssl exporter --- roles/monitoring/prometheus/exporter/TODO | 3 - roles/monitoring/prometheus/exporter/meta/main.yml | 2 + .../prometheus/exporter/ssl/defaults/main.yml | 16 +++ .../prometheus/exporter/ssl/handlers/main.yml | 10 ++ .../prometheus/exporter/ssl/tasks/main.yml | 42 ++++++++ .../exporter/ssl/templates/config.yml.j2 | 4 + .../prometheus/exporter/ssl/templates/service.j2 | 30 ++++++ .../prometheus/server/defaults/main/main.yml | 2 + .../server/defaults/main/rules_blackbox__probe.yml | 11 +- .../prometheus/server/defaults/main/rules_ssl.yml | 3 + .../server/defaults/main/rules_ssl__probe.yml | 119 +++++++++++++++++++++ .../server/templates/targets/ssl/probe.yml.j2 | 5 + 12 files changed, 239 insertions(+), 8 deletions(-) create mode 100644 roles/monitoring/prometheus/exporter/ssl/defaults/main.yml create mode 100644 roles/monitoring/prometheus/exporter/ssl/handlers/main.yml create mode 100644 roles/monitoring/prometheus/exporter/ssl/tasks/main.yml create mode 100644 roles/monitoring/prometheus/exporter/ssl/templates/config.yml.j2 create mode 100644 roles/monitoring/prometheus/exporter/ssl/templates/service.j2 create mode 100644 roles/monitoring/prometheus/server/defaults/main/rules_ssl.yml create mode 100644 roles/monitoring/prometheus/server/defaults/main/rules_ssl__probe.yml create mode 100644 roles/monitoring/prometheus/server/templates/targets/ssl/probe.yml.j2 (limited to 'roles/monitoring/prometheus') diff --git a/roles/monitoring/prometheus/exporter/TODO b/roles/monitoring/prometheus/exporter/TODO index 57179464..53ded412 100644 --- a/roles/monitoring/prometheus/exporter/TODO +++ b/roles/monitoring/prometheus/exporter/TODO @@ -25,6 +25,3 @@ SNMP Exporter: Process Exporter: - https://github.com/ncabatoff/process-exporter - https://packages.debian.org/bullseye/prometheus-process-exporter - -SSL Exporter: - - https://github.com/ribbybibby/ssl_exporter diff --git a/roles/monitoring/prometheus/exporter/meta/main.yml b/roles/monitoring/prometheus/exporter/meta/main.yml index 68fce6cb..31dbd611 100644 --- a/roles/monitoring/prometheus/exporter/meta/main.yml +++ b/roles/monitoring/prometheus/exporter/meta/main.yml @@ -11,3 +11,5 @@ dependencies: when: "'node' in (prometheus_exporters_default | union(prometheus_exporters_extra))" - role: monitoring/prometheus/exporter/nut when: "'nut' in (prometheus_exporters_default | union(prometheus_exporters_extra))" + - role: monitoring/prometheus/exporter/ssl + when: "'ssl' in (prometheus_exporters_default | union(prometheus_exporters_extra))" diff --git a/roles/monitoring/prometheus/exporter/ssl/defaults/main.yml b/roles/monitoring/prometheus/exporter/ssl/defaults/main.yml new file mode 100644 index 00000000..d7edd3f4 --- /dev/null +++ b/roles/monitoring/prometheus/exporter/ssl/defaults/main.yml @@ -0,0 +1,16 @@ +--- +prometheus_exporter_ssl_modules: + tcp: + prober: tcp + http: + prober: https + https: + prober: https + file: + prober: file + kubernetes: + prober: kubernetes + kubeconfig: + prober: kubeconfig + +prometheus_exporter_ssl_modules_extra: {} diff --git a/roles/monitoring/prometheus/exporter/ssl/handlers/main.yml b/roles/monitoring/prometheus/exporter/ssl/handlers/main.yml new file mode 100644 index 00000000..2fb43f19 --- /dev/null +++ b/roles/monitoring/prometheus/exporter/ssl/handlers/main.yml @@ -0,0 +1,10 @@ +--- +- name: restart prometheus-ssl-exporter + service: + name: prometheus-ssl-exporter + state: restarted + +- name: reload nginx + service: + name: nginx + state: reloaded diff --git a/roles/monitoring/prometheus/exporter/ssl/tasks/main.yml b/roles/monitoring/prometheus/exporter/ssl/tasks/main.yml new file mode 100644 index 00000000..c57ea0b1 --- /dev/null +++ b/roles/monitoring/prometheus/exporter/ssl/tasks/main.yml @@ -0,0 +1,42 @@ +--- + ## TODO: pin version +- name: install apt packages + apt: + name: prom-exporter-ssl + state: present + +- name: create config directory + file: + path: /etc/prometheus/exporter/ssl + state: directory + +- name: generate configuration + template: + src: config.yml.j2 + dest: /etc/prometheus/exporter/ssl/config.yml + notify: restart prometheus-ssl-exporter + +- name: generate systemd service unit + template: + src: service.j2 + dest: /etc/systemd/system/prometheus-ssl-exporter.service + notify: restart prometheus-ssl-exporter + +- name: make sure prometheus-ssl-exporter is enabled and started + systemd: + name: prometheus-ssl-exporter.service + daemon_reload: yes + state: started + enabled: yes + +- name: register exporter + copy: + content: | + location = /ssl { + proxy_pass http://127.0.0.1:9219/metrics; + } + location = /ssl/probe { + proxy_pass http://127.0.0.1:9219/probe; + } + dest: /etc/prometheus/exporter/ssl.locations + notify: reload nginx diff --git a/roles/monitoring/prometheus/exporter/ssl/templates/config.yml.j2 b/roles/monitoring/prometheus/exporter/ssl/templates/config.yml.j2 new file mode 100644 index 00000000..1ef84541 --- /dev/null +++ b/roles/monitoring/prometheus/exporter/ssl/templates/config.yml.j2 @@ -0,0 +1,4 @@ +# {{ ansible_managed }} + +modules: + {{ prometheus_exporter_ssl_modules | combine(prometheus_exporter_ssl_modules_extra) | to_nice_yaml(indent=2) | indent(2) }} diff --git a/roles/monitoring/prometheus/exporter/ssl/templates/service.j2 b/roles/monitoring/prometheus/exporter/ssl/templates/service.j2 new file mode 100644 index 00000000..fdd754a4 --- /dev/null +++ b/roles/monitoring/prometheus/exporter/ssl/templates/service.j2 @@ -0,0 +1,30 @@ +[Unit] +Description=Prometheus ssl exporter + +[Service] +Restart=always +ExecStart=/usr/bin/prometheus-ssl-exporter --web.listen-address="127.0.0.1:9219" --config.file=/etc/prometheus/exporter/ssl/config.yml +ExecReload=/bin/kill -HUP $MAINPID + +# systemd hardening-options +AmbientCapabilities= +CapabilityBoundingSet= +DeviceAllow=/dev/null rw +DevicePolicy=strict +LockPersonality=true +MemoryDenyWriteExecute=true +NoNewPrivileges=true +PrivateDevices=true +PrivateTmp=true +ProtectControlGroups=true +ProtectHome=true +ProtectKernelModules=true +ProtectKernelTunables=true +ProtectSystem=strict +RemoveIPC=true +RestrictNamespaces=true +RestrictRealtime=true +SystemCallArchitectures=native + +[Install] +WantedBy=multi-user.target diff --git a/roles/monitoring/prometheus/server/defaults/main/main.yml b/roles/monitoring/prometheus/server/defaults/main/main.yml index 1e0ccf78..3aea0509 100644 --- a/roles/monitoring/prometheus/server/defaults/main/main.yml +++ b/roles/monitoring/prometheus/server/defaults/main/main.yml @@ -20,6 +20,8 @@ prometheus_server_rules: blackbox/probe: "{{ prometheus_server_rules_blackbox__probe + prometheus_server_rules_blackbox__probe_extra }}" ipmi: "{{ prometheus_server_rules_ipmi + prometheus_server_rules_ipmi_extra }}" ipmi/remote: "{{ prometheus_server_rules_ipmi__remote + prometheus_server_rules_ipmi__remote_extra }}" + ssl: "{{ prometheus_server_rules_ssl + prometheus_server_rules_ssl_extra }}" + ssl/probe: "{{ prometheus_server_rules_ssl__probe + prometheus_server_rules_ssl__probe_extra }}" # prometheus_server_alertmanager: # url: "127.0.0.1:9093" diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_blackbox__probe.yml b/roles/monitoring/prometheus/server/defaults/main/rules_blackbox__probe.yml index 9f9d2292..5a75abf0 100644 --- a/roles/monitoring/prometheus/server/defaults/main/rules_blackbox__probe.yml +++ b/roles/monitoring/prometheus/server/defaults/main/rules_blackbox__probe.yml @@ -1,4 +1,5 @@ --- +## https://awesome-prometheus-alerts.grep.to/rules#blackbox prometheus_server_rules_blackbox__probe_extra: [] prometheus_server_rules_blackbox__probe: - alert: BlackboxProbeFailed @@ -26,16 +27,16 @@ prometheus_server_rules_blackbox__probe: severity: warning annotations: summary: Blackbox SSL certificate will expire soon (instance {{ '{{' }} $labels.instance {{ '}}' }}) - description: "SSL certificate expires in 30 days\n VALUE = {{ '{{' }} $value {{ '}}' }}\n LABELS = {{ '{{' }} $labels {{ '}}' }}" + description: "SSL certificate expires in less than 30 days\n VALUE = {{ '{{' }} $value {{ '}}' }}\n LABELS = {{ '{{' }} $labels {{ '}}' }}" - - alert: BlackboxSslCertificateWillExpireSoon - expr: probe_ssl_earliest_cert_expiry - time() < 86400 * 3 + - alert: BlackboxSslCertificateWillExpireVerySoon + expr: probe_ssl_earliest_cert_expiry - time() < 86400 * 7 for: 0m labels: severity: critical annotations: - summary: Blackbox SSL certificate will expire soon (instance {{ '{{' }} $labels.instance {{ '}}' }}) - description: "SSL certificate expires in 3 days\n VALUE = {{ '{{' }} $value {{ '}}' }}\n LABELS = {{ '{{' }} $labels {{ '}}' }}" + summary: Blackbox SSL certificate will expire very soon (instance {{ '{{' }} $labels.instance {{ '}}' }}) + description: "SSL certificate expires in less than 7 days\n VALUE = {{ '{{' }} $value {{ '}}' }}\n LABELS = {{ '{{' }} $labels {{ '}}' }}" - alert: BlackboxSslCertificateExpired expr: probe_ssl_earliest_cert_expiry - time() <= 0 diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_ssl.yml b/roles/monitoring/prometheus/server/defaults/main/rules_ssl.yml new file mode 100644 index 00000000..06c7e027 --- /dev/null +++ b/roles/monitoring/prometheus/server/defaults/main/rules_ssl.yml @@ -0,0 +1,3 @@ +--- +prometheus_server_rules_ssl_extra: [] +prometheus_server_rules_ssl: [] diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_ssl__probe.yml b/roles/monitoring/prometheus/server/defaults/main/rules_ssl__probe.yml new file mode 100644 index 00000000..8805de0a --- /dev/null +++ b/roles/monitoring/prometheus/server/defaults/main/rules_ssl__probe.yml @@ -0,0 +1,119 @@ +--- +prometheus_server_rules_ssl__probe_extra: [] +prometheus_server_rules_ssl__probe: + - alert: SslCertificateProbeFailed + expr: ssl_probe_success == 0 + for: 0m + labels: + severity: critical + annotations: + summary: SSL certificate probe failed (instance {{ '{{' }} $labels.instance {{ '}}' }}) + description: "Failed to fetch SSL certificate information {{ '{{' }} $labels.instance {{ '}}' }}\n VALUE = {{ '{{' }} $value {{ '}}' }}\n LABELS = {{ '{{' }} $labels {{ '}}' }}" + + - alert: SslCertificateExpiresSoon + expr: ssl_cert_not_after - time() < 86400 * 30 + for: 0m + labels: + severity: warning + annotations: + summary: SSL certificate expires soon (instance {{ '{{' }} $labels.instance {{ '}}' }}) + description: "The certificate {{ '{{' }} $labels.cn {{ '}}' }} is expiring in less than 30 days\n VALUE = {{ '{{' }} $value {{ '}}' }}\n LABELS = BRACEOPEN $labels {{ '}}' }}" + + - alert: SslCertificateExpiresVerySoon + expr: ssl_cert_not_after - time() < 86400 * 7 + for: 0m + labels: + severity: critical + annotations: + summary: SSL certificate expires soon (instance {{ '{{' }} $labels.instance {{ '}}' }}) + description: "The certificate {{ '{{' }} $labels.cn {{ '}}' }} is expiring in less than 7 days\n VALUE = {{ '{{' }} $value {{ '}}' }}\n LABELS = BRACEOPEN $labels {{ '}}' }}" + + - alert: SslCertificateChainExpiresSoon + expr: ssl_verified_cert_not_after - time() < 86400 * 30 + for: 0m + labels: + severity: warning + annotations: + summary: SSL chain certificate expires soon (instance {{ '{{' }} $labels.instance {{ '}}' }}) + description: "The chain certificate {{ '{{' }} $labels.cn {{ '}}' }} is expiring in less than 30 days\n VALUE = {{ '{{' }} $value {{ '}}' }}\n LABELS = BRACEOPEN $labels {{ '}}' }}" + + - alert: SslCertificateChainExpiresVerySoon + expr: ssl_verified_cert_not_after - time() < 86400 * 7 + for: 0m + labels: + severity: critical + annotations: + summary: SSL chain certificate expires soon (instance {{ '{{' }} $labels.instance {{ '}}' }}) + description: "The chain certificate {{ '{{' }} $labels.cn {{ '}}' }} is expiring in less than 7 days\n VALUE = {{ '{{' }} $value {{ '}}' }}\n LABELS = BRACEOPEN $labels {{ '}}' }}" + + - alert: SslCertificateOscpStatusUnknown + expr: ssl_ocsp_response_status == 2 + for: 0m + labels: + severity: warning + annotations: + summary: SSL certificate OSCP status unknown (instance {{ '{{' }} $labels.instance {{ '}}' }}) + description: "Failed to get the OSCP status for {{ '{{' }} $labels.cn {{ '}}' }}\n VALUE = {{ '{{' }} $value {{ '}}' }}\n LABELS = {{ '{{' }} $labels {{ '}}' }}" + + - alert: SslCertificateOscpStatusRevoked + expr: ssl_ocsp_response_status == 1 + for: 0m + labels: + severity: critical + annotations: + summary: SSL certificate revoked (instance {{ '{{' }} $labels.instance {{ '}}' }}) + description: "OCSP reports SSL certificate {{ '{{' }} $labels.cn {{ '}}' }} as revoked\n VALUE = {{ '{{' }} $value {{ '}}' }}\n LABELS = {{ '{{' }} $labels {{ '}}' }}" + + - alert: SslCertificateFileExpiresSoon + expr: ssl_file_cert_not_after - time() < 86400 * 30 + for: 0m + labels: + severity: warning + annotations: + summary: SSL certificate expires soon (instance {{ '{{' }} $labels.instance {{ '}}' }}) + description: "The certificate in {{ '{{' }} $labels.file {{ '}}' }} is expiring in less than 30 days\n VALUE = {{ '{{' }} $value {{ '}}' }}\n LABELS = BRACEOPEN $labels {{ '}}' }}" + + - alert: SslCertificateFileExpiresVerySoon + expr: ssl_file_cert_not_after - time() < 86400 * 7 + for: 0m + labels: + severity: critical + annotations: + summary: SSL certificate expires soon (instance {{ '{{' }} $labels.instance {{ '}}' }}) + description: "The certificate in {{ '{{' }} $labels.file {{ '}}' }} is expiring in less than 7 days\n VALUE = {{ '{{' }} $value {{ '}}' }}\n LABELS = BRACEOPEN $labels {{ '}}' }}" + + - alert: SslCertificateKubernetesExpiresSoon + expr: ssl_kubernetes_cert_not_after - time() < 86400 * 30 + for: 0m + labels: + severity: warning + annotations: + summary: SSL kubernetes certificate expires soon (instance {{ '{{' }} $labels.instance {{ '}}' }}) + description: "The certificate in kubernetes secret {{ '{{' }} $labels.namespace {{ '}}' }}/{{ '{{' }} $labels.secret {{ '}}' }} is expiring in less than 30 days\n VALUE = {{ '{{' }} $value {{ '}}' }}\n LABELS = BRACEOPEN $labels {{ '}}' }}" + + - alert: SslCertificateKubernetesExpiresVerySoon + expr: ssl_kubernetes_cert_not_after - time() < 86400 * 7 + for: 0m + labels: + severity: critical + annotations: + summary: SSL kubernetes certificate expires soon (instance {{ '{{' }} $labels.instance {{ '}}' }}) + description: "The certificate in kubernetes secret {{ '{{' }} $labels.namespace {{ '}}' }}/{{ '{{' }} $labels.secret {{ '}}' }} is expiring in less than 7 days\n VALUE = {{ '{{' }} $value {{ '}}' }}\n LABELS = BRACEOPEN $labels {{ '}}' }}" + + - alert: SslCertificateKubeconfigExpiresSoon + expr: ssl_kubeconfig_cert_not_after - time() < 86400 * 30 + for: 0m + labels: + severity: warning + annotations: + summary: SSL kubeconfig certificate expires soon (instance {{ '{{' }} $labels.instance {{ '}}' }}) + description: "The certificate in kubeconfig {{ '{{' }} $labels.kubeconfig {{ '}}' }} is expiring in less than 30 days\n VALUE = {{ '{{' }} $value {{ '}}' }}\n LABELS = BRACEOPEN $labels {{ '}}' }}" + + - alert: SslCertificateKubeconfigExpiresVerySoon + expr: ssl_kubeconfig_cert_not_after - time() < 86400 * 7 + for: 0m + labels: + severity: critical + annotations: + summary: SSL kubeconfig certificate expires soon (instance {{ '{{' }} $labels.instance {{ '}}' }}) + description: "The certificate in kubeconfig {{ '{{' }} $labels.kubeconfig {{ '}}' }} is expiring in less than 7 days\n VALUE = {{ '{{' }} $value {{ '}}' }}\n LABELS = BRACEOPEN $labels {{ '}}' }}" diff --git a/roles/monitoring/prometheus/server/templates/targets/ssl/probe.yml.j2 b/roles/monitoring/prometheus/server/templates/targets/ssl/probe.yml.j2 new file mode 100644 index 00000000..4e336873 --- /dev/null +++ b/roles/monitoring/prometheus/server/templates/targets/ssl/probe.yml.j2 @@ -0,0 +1,5 @@ +- targets: [ '{{ hostvars[target.exporter_hostname].prometheus_scrape_endpoint }}' ] + labels: + instance: '{{ target.instance }}' + __param_target: '{{ target.config.target }}' + __param_module: '{{ target.config.module }}' -- cgit v1.2.3