From 2ad12c3b7e7f6f2159bec33105448e6114ffdfec Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Sun, 29 Aug 2021 22:41:57 +0200 Subject: prometheus: move blackbox job config to new setup --- inventory/group_vars/chaos-at-home-ups/vars.yml | 8 +- .../group_vars/promzone-chaos-at-home/vars.yml | 3 + inventory/host_vars/ch-mon.yml | 86 +++++----------------- .../prometheus/server/defaults/main/main.yml | 3 + .../server/defaults/main/rules_blackbox-https.yml | 3 + .../server/defaults/main/rules_blackbox-ping.yml | 3 + .../server/defaults/main/rules_blackbox-ssh.yml | 3 + .../prometheus/server/filter_plugins/prometheus.py | 5 +- .../templates/job-snippets/blackbox-https.j2 | 14 ++++ .../server/templates/job-snippets/blackbox-ping.j2 | 14 ++++ .../server/templates/job-snippets/blackbox-ssh.j2 | 14 ++++ .../server/templates/targets/blackbox-https.yml.j2 | 4 + .../server/templates/targets/blackbox-ping.yml.j2 | 4 + .../server/templates/targets/blackbox-ssh.yml.j2 | 4 + 14 files changed, 94 insertions(+), 74 deletions(-) create mode 100644 roles/monitoring/prometheus/server/defaults/main/rules_blackbox-https.yml create mode 100644 roles/monitoring/prometheus/server/defaults/main/rules_blackbox-ping.yml create mode 100644 roles/monitoring/prometheus/server/defaults/main/rules_blackbox-ssh.yml create mode 100644 roles/monitoring/prometheus/server/templates/job-snippets/blackbox-https.j2 create mode 100644 roles/monitoring/prometheus/server/templates/job-snippets/blackbox-ping.j2 create mode 100644 roles/monitoring/prometheus/server/templates/job-snippets/blackbox-ssh.j2 create mode 100644 roles/monitoring/prometheus/server/templates/targets/blackbox-https.yml.j2 create mode 100644 roles/monitoring/prometheus/server/templates/targets/blackbox-ping.yml.j2 create mode 100644 roles/monitoring/prometheus/server/templates/targets/blackbox-ssh.yml.j2 diff --git a/inventory/group_vars/chaos-at-home-ups/vars.yml b/inventory/group_vars/chaos-at-home-ups/vars.yml index cc7ae585..5ff68452 100644 --- a/inventory/group_vars/chaos-at-home-ups/vars.yml +++ b/inventory/group_vars/chaos-at-home-ups/vars.yml @@ -12,7 +12,7 @@ prometheus_exporters_default: - openwrt prometheus_special_job_nut: - exporter_hostname: ch-mon - instance: "ups-{{ ups_name }}" - ups: "{{ ups_name }}" - server: "{{ network_mgmt_zone.prefix | ipaddr(network_mgmt_zone.offsets[inventory_hostname]) | ipaddr('address') }}" + - exporter_hostname: ch-mon + instance: "ups-{{ ups_name }}" + ups: "{{ ups_name }}" + server: "{{ network_mgmt_zone.prefix | ipaddr(network_mgmt_zone.offsets[inventory_hostname]) | ipaddr('address') }}" diff --git a/inventory/group_vars/promzone-chaos-at-home/vars.yml b/inventory/group_vars/promzone-chaos-at-home/vars.yml index a48cbad7..fcb04716 100644 --- a/inventory/group_vars/promzone-chaos-at-home/vars.yml +++ b/inventory/group_vars/promzone-chaos-at-home/vars.yml @@ -11,6 +11,9 @@ prometheus_server_jobs_generic: - openwrt prometheus_server_jobs_special: - nut + - blackbox-ping + - blackbox-https + - blackbox-ssh prometheus_zone_name: chaos@home prometheus_zone_targets: "{{ groups['promzone-chaos-at-home'] }}" diff --git a/inventory/host_vars/ch-mon.yml b/inventory/host_vars/ch-mon.yml index 1a050b6a..7d8e334b 100644 --- a/inventory/host_vars/ch-mon.yml +++ b/inventory/host_vars/ch-mon.yml @@ -76,6 +76,24 @@ prometheus_exporter_blackbox_modules_extra: icmp: prober: icmp +prometheus_special_job_blackbox_ping: + - exporter_hostname: ch-mon + instance: "ping-magentagw" + address: 62.99.185.129 + - exporter_hostname: ch-mon + instance: "ping-quad9" + address: 9.9.9.9 + +prometheus_special_job_blackbox_https: + - exporter_hostname: ch-mon + instance: "https-web.chaos-at-home.org" + address: web.chaos-at-home.org + +prometheus_special_job_blackbox_ssh: + - exporter_hostname: ch-mon + instance: "ssh-{{ inventory_hostname }}" + address: "{{ network_zones.svc.prefix | ipaddr(network_zones.svc.offsets[inventory_hostname]) | ipaddr('address') }}:{{ ansible_port | default(22) }}" + promethues_alertmanager_smtp: smarthost: "{{ network_zones.lan.prefix | ipaddr(network_zones.lan.offsets['ch-prometheus-legacy']) | ipaddr('address') }}:25" @@ -86,71 +104,3 @@ prometheus_alertmanager_web_route_prefix: /alertmanager/ grafana_secret_key: "{{ vault_grafana_secret_key }}" - -prometheus_server_jobs_extra: | - - job_name: 'ping' - metrics_path: /proxy - params: - module: - - blackbox - - icmp - scheme: https - tls_config: - ca_file: /etc/ssl/prometheus/ca-crt.pem - cert_file: /etc/ssl/prometheus/server/scrape-crt.pem - key_file: /etc/ssl/prometheus/server/scrape-key.pem - static_configs: - - targets: - - 62.99.185.129 - - 9.9.9.9 - relabel_configs: - - source_labels: [__address__] - target_label: __param_target - - source_labels: [__param_target] - target_label: instance - - target_label: __address__ - replacement: 192.168.32.230:9999 - - - job_name: 'https' - metrics_path: /proxy - params: - module: - - blackbox - - http_tls_2xx - scheme: https - tls_config: - ca_file: /etc/ssl/prometheus/ca-crt.pem - cert_file: /etc/ssl/prometheus/server/scrape-crt.pem - key_file: /etc/ssl/prometheus/server/scrape-key.pem - static_configs: - - targets: - - web.chaos-at-home.org - relabel_configs: - - source_labels: [__address__] - target_label: __param_target - - source_labels: [__param_target] - target_label: instance - - target_label: __address__ - replacement: 192.168.32.230:9999 - - - job_name: 'ssh' - metrics_path: /proxy - params: - module: - - blackbox - - ssh_banner - scheme: https - tls_config: - ca_file: /etc/ssl/prometheus/ca-crt.pem - cert_file: /etc/ssl/prometheus/server/scrape-crt.pem - key_file: /etc/ssl/prometheus/server/scrape-key.pem - static_configs: - - targets: - - 192.168.32.230:222 - relabel_configs: - - source_labels: [__address__] - target_label: __param_target - - target_label: instance - replacement: 'ch-mon' - - target_label: __address__ - replacement: 192.168.32.230:9999 diff --git a/roles/monitoring/prometheus/server/defaults/main/main.yml b/roles/monitoring/prometheus/server/defaults/main/main.yml index f6679fa7..95b9da6d 100644 --- a/roles/monitoring/prometheus/server/defaults/main/main.yml +++ b/roles/monitoring/prometheus/server/defaults/main/main.yml @@ -16,6 +16,9 @@ prometheus_server_rules: node: "{{ prometheus_server_rules_node + prometheus_server_rules_node_extra }}" openwrt: "{{ prometheus_server_rules_openwrt + prometheus_server_rules_node_extra }}" nut: "{{ prometheus_server_rules_nut + prometheus_server_rules_nut_extra }}" + "blackbox-ping": "{{ prometheus_server_rules_blackbox_ping + prometheus_server_rules_blackbox_ping_extra }}" + "blackbox-https": "{{ prometheus_server_rules_blackbox_https + prometheus_server_rules_blackbox_https_extra }}" + "blackbox-ssh": "{{ prometheus_server_rules_blackbox_ssh + prometheus_server_rules_blackbox_ssh_extra }}" # prometheus_server_alertmanager: # url: "127.0.0.1:9093" diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_blackbox-https.yml b/roles/monitoring/prometheus/server/defaults/main/rules_blackbox-https.yml new file mode 100644 index 00000000..bb806075 --- /dev/null +++ b/roles/monitoring/prometheus/server/defaults/main/rules_blackbox-https.yml @@ -0,0 +1,3 @@ +--- +prometheus_server_rules_blackbox_https_extra: [] +prometheus_server_rules_blackbox_https: [] diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_blackbox-ping.yml b/roles/monitoring/prometheus/server/defaults/main/rules_blackbox-ping.yml new file mode 100644 index 00000000..56c122f5 --- /dev/null +++ b/roles/monitoring/prometheus/server/defaults/main/rules_blackbox-ping.yml @@ -0,0 +1,3 @@ +--- +prometheus_server_rules_blackbox_ping_extra: [] +prometheus_server_rules_blackbox_ping: [] diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_blackbox-ssh.yml b/roles/monitoring/prometheus/server/defaults/main/rules_blackbox-ssh.yml new file mode 100644 index 00000000..727d2292 --- /dev/null +++ b/roles/monitoring/prometheus/server/defaults/main/rules_blackbox-ssh.yml @@ -0,0 +1,3 @@ +--- +prometheus_server_rules_blackbox_ssh_extra: [] +prometheus_server_rules_blackbox_ssh: [] diff --git a/roles/monitoring/prometheus/server/filter_plugins/prometheus.py b/roles/monitoring/prometheus/server/filter_plugins/prometheus.py index 6e18481c..5a8722c2 100644 --- a/roles/monitoring/prometheus/server/filter_plugins/prometheus.py +++ b/roles/monitoring/prometheus/server/filter_plugins/prometheus.py @@ -23,9 +23,10 @@ def prometheus_special_job_targets(hostvars, jobs, targets): result = [] for job in jobs: for target in targets: - config_varname = 'prometheus_special_job_' + job + config_varname = 'prometheus_special_job_' + job.replace('-', '_') if config_varname in hostvars[target]: - result.append({'job': job, 'instance': hostvars[target][config_varname]['instance'], 'config': hostvars[target][config_varname]}) + for config in hostvars[target][config_varname]: + result.append({'job': job, 'instance': config['instance'], 'config': config}) return result except Exception as e: raise errors.AnsibleFilterError("prometheus_special_job_targets(): %s" % str(e)) diff --git a/roles/monitoring/prometheus/server/templates/job-snippets/blackbox-https.j2 b/roles/monitoring/prometheus/server/templates/job-snippets/blackbox-https.j2 new file mode 100644 index 00000000..0a6d2dfa --- /dev/null +++ b/roles/monitoring/prometheus/server/templates/job-snippets/blackbox-https.j2 @@ -0,0 +1,14 @@ + - job_name: '{{ job }}' + metrics_path: /proxy + params: + module: + - blackbox + - http_tls_2xx + scheme: https + tls_config: + ca_file: /etc/ssl/prometheus/ca-crt.pem + cert_file: /etc/ssl/prometheus/server/scrape-crt.pem + key_file: /etc/ssl/prometheus/server/scrape-key.pem + file_sd_configs: + - files: + - "/etc/prometheus/jobs/{{ job }}/*.yml" diff --git a/roles/monitoring/prometheus/server/templates/job-snippets/blackbox-ping.j2 b/roles/monitoring/prometheus/server/templates/job-snippets/blackbox-ping.j2 new file mode 100644 index 00000000..7f4f12df --- /dev/null +++ b/roles/monitoring/prometheus/server/templates/job-snippets/blackbox-ping.j2 @@ -0,0 +1,14 @@ + - job_name: '{{ job }}' + metrics_path: /proxy + params: + module: + - blackbox + - icmp + scheme: https + tls_config: + ca_file: /etc/ssl/prometheus/ca-crt.pem + cert_file: /etc/ssl/prometheus/server/scrape-crt.pem + key_file: /etc/ssl/prometheus/server/scrape-key.pem + file_sd_configs: + - files: + - "/etc/prometheus/jobs/{{ job }}/*.yml" diff --git a/roles/monitoring/prometheus/server/templates/job-snippets/blackbox-ssh.j2 b/roles/monitoring/prometheus/server/templates/job-snippets/blackbox-ssh.j2 new file mode 100644 index 00000000..18381e32 --- /dev/null +++ b/roles/monitoring/prometheus/server/templates/job-snippets/blackbox-ssh.j2 @@ -0,0 +1,14 @@ + - job_name: '{{ job }}' + metrics_path: /proxy + params: + module: + - blackbox + - ssh_banner + scheme: https + tls_config: + ca_file: /etc/ssl/prometheus/ca-crt.pem + cert_file: /etc/ssl/prometheus/server/scrape-crt.pem + key_file: /etc/ssl/prometheus/server/scrape-key.pem + file_sd_configs: + - files: + - "/etc/prometheus/jobs/{{ job }}/*.yml" diff --git a/roles/monitoring/prometheus/server/templates/targets/blackbox-https.yml.j2 b/roles/monitoring/prometheus/server/templates/targets/blackbox-https.yml.j2 new file mode 100644 index 00000000..e843de36 --- /dev/null +++ b/roles/monitoring/prometheus/server/templates/targets/blackbox-https.yml.j2 @@ -0,0 +1,4 @@ +- targets: [ "{{ hostvars[target.config.exporter_hostname].prometheus_scrape_endpoint }}" ] + labels: + instance: "{{ target.instance }}" + __param_target: {{ target.config.address }} diff --git a/roles/monitoring/prometheus/server/templates/targets/blackbox-ping.yml.j2 b/roles/monitoring/prometheus/server/templates/targets/blackbox-ping.yml.j2 new file mode 100644 index 00000000..e843de36 --- /dev/null +++ b/roles/monitoring/prometheus/server/templates/targets/blackbox-ping.yml.j2 @@ -0,0 +1,4 @@ +- targets: [ "{{ hostvars[target.config.exporter_hostname].prometheus_scrape_endpoint }}" ] + labels: + instance: "{{ target.instance }}" + __param_target: {{ target.config.address }} diff --git a/roles/monitoring/prometheus/server/templates/targets/blackbox-ssh.yml.j2 b/roles/monitoring/prometheus/server/templates/targets/blackbox-ssh.yml.j2 new file mode 100644 index 00000000..e843de36 --- /dev/null +++ b/roles/monitoring/prometheus/server/templates/targets/blackbox-ssh.yml.j2 @@ -0,0 +1,4 @@ +- targets: [ "{{ hostvars[target.config.exporter_hostname].prometheus_scrape_endpoint }}" ] + labels: + instance: "{{ target.instance }}" + __param_target: {{ target.config.address }} -- cgit v1.2.3