diff options
24 files changed, 78 insertions, 72 deletions
diff --git a/inventory/group_vars/chaos-at-home-ups/vars.yml b/inventory/group_vars/chaos-at-home-ups/vars.yml index 5ff68452..4f003a7a 100644 --- a/inventory/group_vars/chaos-at-home-ups/vars.yml +++ b/inventory/group_vars/chaos-at-home-ups/vars.yml @@ -11,7 +11,7 @@ prometheus_scrape_endpoint: "{{ network_mgmt_zone.prefix | ipaddr(network_mgmt_z prometheus_exporters_default: - openwrt -prometheus_special_job_nut: +prometheus_special_job_nut_ups: - exporter_hostname: ch-mon instance: "ups-{{ ups_name }}" ups: "{{ ups_name }}" diff --git a/inventory/group_vars/ele-ups/vars.yml b/inventory/group_vars/ele-ups/vars.yml index cbee3ee8..a57382ff 100644 --- a/inventory/group_vars/ele-ups/vars.yml +++ b/inventory/group_vars/ele-ups/vars.yml @@ -14,7 +14,7 @@ prometheus_scrape_endpoint: "{{ network_mgmt_zone.prefix | ipaddr(network_mgmt_z prometheus_exporters_default: - openwrt -prometheus_special_job_nut: +prometheus_special_job_nut_ups: exporter_hostname: ele-mon instance: "ups-{{ ups_name }}" ups: "{{ ups_name }}" diff --git a/inventory/group_vars/promzone-chaos-at-home/vars.yml b/inventory/group_vars/promzone-chaos-at-home/vars.yml index fcb04716..d1958d47 100644 --- a/inventory/group_vars/promzone-chaos-at-home/vars.yml +++ b/inventory/group_vars/promzone-chaos-at-home/vars.yml @@ -6,11 +6,10 @@ prometheus_exporters_default: - node prometheus_server: ch-mon -prometheus_server_jobs_generic: +prometheus_server_jobs: - node - openwrt -prometheus_server_jobs_special: - - nut + - nut-ups - blackbox-ping - blackbox-https - blackbox-ssh diff --git a/inventory/group_vars/promzone-elevate-festival/vars.yml b/inventory/group_vars/promzone-elevate-festival/vars.yml index e94943d7..a65a0cb7 100644 --- a/inventory/group_vars/promzone-elevate-festival/vars.yml +++ b/inventory/group_vars/promzone-elevate-festival/vars.yml @@ -6,9 +6,10 @@ prometheus_exporters_default: - node prometheus_server: ele-mon -prometheus_server_jobs_generic: +prometheus_server_jobs: - node - openwrt + - nut-ups prometheus_zone_name: Elevate Festival prometheus_zone_targets: "{{ groups['promzone-elevate-festival'] }}" diff --git a/roles/monitoring/prometheus/alertmanager/tasks/main.yml b/roles/monitoring/prometheus/alertmanager/tasks/main.yml index fe8ce9ca..0dce6ef4 100644 --- a/roles/monitoring/prometheus/alertmanager/tasks/main.yml +++ b/roles/monitoring/prometheus/alertmanager/tasks/main.yml @@ -6,6 +6,7 @@ - spreadspace_apt_repo_components is defined - "'prometheus' in spreadspace_apt_repo_components" + ## TODO: pin version - name: install apt packages apt: name: prom-alertmanager diff --git a/roles/monitoring/prometheus/exporter/base/tasks/main.yml b/roles/monitoring/prometheus/exporter/base/tasks/main.yml index 9a214f39..9e35c48d 100644 --- a/roles/monitoring/prometheus/exporter/base/tasks/main.yml +++ b/roles/monitoring/prometheus/exporter/base/tasks/main.yml @@ -6,6 +6,7 @@ - spreadspace_apt_repo_components is defined - "'prometheus' in spreadspace_apt_repo_components" + ## TODO: pin version - name: install apt packages apt: name: prom-exporter-exporter diff --git a/roles/monitoring/prometheus/exporter/blackbox/tasks/main.yml b/roles/monitoring/prometheus/exporter/blackbox/tasks/main.yml index 7ecd8113..96c247ec 100644 --- a/roles/monitoring/prometheus/exporter/blackbox/tasks/main.yml +++ b/roles/monitoring/prometheus/exporter/blackbox/tasks/main.yml @@ -1,4 +1,5 @@ --- + ## TODO: pin version - name: install apt packages apt: name: prom-exporter-blackbox diff --git a/roles/monitoring/prometheus/exporter/mikrotik/tasks/main.yml b/roles/monitoring/prometheus/exporter/mikrotik/tasks/main.yml index c3ffe31b..dda33e9f 100644 --- a/roles/monitoring/prometheus/exporter/mikrotik/tasks/main.yml +++ b/roles/monitoring/prometheus/exporter/mikrotik/tasks/main.yml @@ -1,4 +1,5 @@ --- + ## TODO: pin version - name: install apt packages apt: name: prom-exporter-mikrotik diff --git a/roles/monitoring/prometheus/exporter/node/tasks/main.yml b/roles/monitoring/prometheus/exporter/node/tasks/main.yml index 8392e580..00a4ab3f 100644 --- a/roles/monitoring/prometheus/exporter/node/tasks/main.yml +++ b/roles/monitoring/prometheus/exporter/node/tasks/main.yml @@ -1,4 +1,5 @@ --- + ## TODO: pin version - name: install apt packages apt: name: prom-exporter-node diff --git a/roles/monitoring/prometheus/exporter/nut/tasks/main.yml b/roles/monitoring/prometheus/exporter/nut/tasks/main.yml index 519ac7a0..78a8e817 100644 --- a/roles/monitoring/prometheus/exporter/nut/tasks/main.yml +++ b/roles/monitoring/prometheus/exporter/nut/tasks/main.yml @@ -1,4 +1,5 @@ --- + ## TODO: pin version - name: install apt packages apt: name: prom-exporter-nut diff --git a/roles/monitoring/prometheus/server/defaults/main/main.yml b/roles/monitoring/prometheus/server/defaults/main/main.yml index 95b9da6d..5be3ecd3 100644 --- a/roles/monitoring/prometheus/server/defaults/main/main.yml +++ b/roles/monitoring/prometheus/server/defaults/main/main.yml @@ -5,9 +5,8 @@ prometheus_server_retention: "15d" -prometheus_server_jobs_generic: +prometheus_server_jobs: - node -prometheus_server_jobs_special: [] #prometheus_server_jobs_extra: | # - job_name: ... @@ -15,7 +14,7 @@ prometheus_server_rules: prometheus: "{{ prometheus_server_rules_prometheus + ((prometheus_server_alertmanager is defined) | ternary(prometheus_server_rules_prometheus_alertmanager, [])) + prometheus_server_rules_prometheus_extra }}" node: "{{ prometheus_server_rules_node + prometheus_server_rules_node_extra }}" openwrt: "{{ prometheus_server_rules_openwrt + prometheus_server_rules_node_extra }}" - nut: "{{ prometheus_server_rules_nut + prometheus_server_rules_nut_extra }}" + "nut-ups": "{{ prometheus_server_rules_nut_ups + prometheus_server_rules_nut_ups_extra }}" "blackbox-ping": "{{ prometheus_server_rules_blackbox_ping + prometheus_server_rules_blackbox_ping_extra }}" "blackbox-https": "{{ prometheus_server_rules_blackbox_https + prometheus_server_rules_blackbox_https_extra }}" "blackbox-ssh": "{{ prometheus_server_rules_blackbox_ssh + prometheus_server_rules_blackbox_ssh_extra }}" diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_nut-ups.yml b/roles/monitoring/prometheus/server/defaults/main/rules_nut-ups.yml new file mode 100644 index 00000000..842007b4 --- /dev/null +++ b/roles/monitoring/prometheus/server/defaults/main/rules_nut-ups.yml @@ -0,0 +1,3 @@ +--- +prometheus_server_rules_nut_ups_extra: [] +prometheus_server_rules_nut_ups: [] diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_nut.yml b/roles/monitoring/prometheus/server/defaults/main/rules_nut.yml deleted file mode 100644 index d8d64f64..00000000 --- a/roles/monitoring/prometheus/server/defaults/main/rules_nut.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -prometheus_server_rules_nut_extra: [] -prometheus_server_rules_nut: [] diff --git a/roles/monitoring/prometheus/server/filter_plugins/prometheus.py b/roles/monitoring/prometheus/server/filter_plugins/prometheus.py index 5a8722c2..ab865f93 100644 --- a/roles/monitoring/prometheus/server/filter_plugins/prometheus.py +++ b/roles/monitoring/prometheus/server/filter_plugins/prometheus.py @@ -6,38 +6,29 @@ from functools import partial from ansible import errors -def prometheus_generic_job_targets(hostvars, jobs, targets): +def prometheus_job_targets(hostvars, jobs, targets): try: result = [] for job in jobs: for target in targets: - enabled = job in hostvars[target]['prometheus_exporters_default'] or job in hostvars[target]['prometheus_exporters_extra'] - result.append({'job': job, 'instance': target, 'enabled': enabled}) + special_config_varname = 'prometheus_special_job_' + job.replace('-', '_') + if special_config_varname in hostvars[target]: + for config in hostvars[target][special_config_varname]: + result.append({'job': job, 'instance': config['instance'], 'config': config, 'enabled': True}) + + else: + enabled = job in hostvars[target]['prometheus_exporters_default'] or job in hostvars[target]['prometheus_exporters_extra'] + result.append({'job': job, 'instance': target, 'enabled': enabled}) return result except Exception as e: - raise errors.AnsibleFilterError("prometheus_generic_job_targets(): %s" % str(e)) - - -def prometheus_special_job_targets(hostvars, jobs, targets): - try: - result = [] - for job in jobs: - for target in targets: - config_varname = 'prometheus_special_job_' + job.replace('-', '_') - if config_varname in hostvars[target]: - for config in hostvars[target][config_varname]: - result.append({'job': job, 'instance': config['instance'], 'config': config}) - return result - except Exception as e: - raise errors.AnsibleFilterError("prometheus_special_job_targets(): %s" % str(e)) + raise errors.AnsibleFilterError("prometheus_job_targets(): %s" % str(e)) class FilterModule(object): ''' prometheus filters ''' filter_map = { - 'prometheus_generic_job_targets': prometheus_generic_job_targets, - 'prometheus_special_job_targets': prometheus_special_job_targets, + 'prometheus_job_targets': prometheus_job_targets, } def filters(self): diff --git a/roles/monitoring/prometheus/server/tasks/main.yml b/roles/monitoring/prometheus/server/tasks/main.yml index d0ccd8af..4bcaa2d5 100644 --- a/roles/monitoring/prometheus/server/tasks/main.yml +++ b/roles/monitoring/prometheus/server/tasks/main.yml @@ -13,6 +13,7 @@ include_role: name: "storage/{{ prometheus_server_storage.type }}/volume" + ## TODO: pin version - name: install apt packages apt: name: prom-server @@ -37,50 +38,46 @@ - name: create configuration directories loop: - - jobs - rules - targets file: path: "/etc/prometheus/{{ item }}" state: directory -- name: create sub-directroy for all exporter types in jobs directory - loop: "{{ prometheus_server_jobs_generic + prometheus_server_jobs_special }}" +- name: create sub-directroy for all jobs in targets directory + loop: "{{ prometheus_server_jobs }}" file: - path: "/etc/prometheus/jobs/{{ item }}" + path: "/etc/prometheus/targets/{{ item }}" state: directory -- name: generate generic targets config - loop: "{{ prometheus_zone_targets }}" - loop_control: - loop_var: target - template: - src: targets/generic.yml.j2 - dest: "/etc/prometheus/targets/{{ target }}.yml" - notify: reload prometheus - -- name: enable targets for generic jobs - loop: "{{ hostvars | prometheus_generic_job_targets(prometheus_server_jobs_generic, prometheus_zone_targets) }}" - loop_control: - label: "{{ item.job }} -> {{ item.instance }}" - file: - src: "{{ item.enabled | ternary('/etc/prometheus/targets/' + item.instance + '.yml', omit) }}" - path: "/etc/prometheus/jobs/{{ item.job }}/{{ item.instance }}.yml" - state: "{{ item.enabled | ternary('link', 'absent') }}" - notify: reload prometheus +- name: enable/disable job targets + vars: + job_targets: "{{ hostvars | prometheus_job_targets(prometheus_server_jobs, prometheus_zone_targets) }}" + block: + - name: install files for enabled targets + loop: "{{ job_targets }}" + loop_control: + loop_var: target + label: "{{ target.job }} -> {{ target.instance }}" + when: target.enabled + template: + src: "{{ lookup('first_found', {'paths': ['templates/targets'], 'files': [target.job + '.yml.j2', 'generic.yml.j2']}) }}" + dest: "/etc/prometheus/targets/{{ target.job }}/{{ target.instance }}.yml" + notify: reload prometheus -- name: enable targets for special jobs - loop: "{{ hostvars | prometheus_special_job_targets(prometheus_server_jobs_special, prometheus_zone_targets) }}" - loop_control: - loop_var: target - label: "{{ target.job }} -> {{ target.instance }}" - template: - src: "targets/{{ target.job }}.yml.j2" - dest: "/etc/prometheus/jobs/{{ target.job }}/{{ target.instance }}.yml" - notify: reload prometheus + - name: remove files for disabled targets + loop: "{{ job_targets }}" + loop_control: + loop_var: target + label: "{{ target.job }} -> {{ target.instance }}" + when: not target.enabled + file: + path: "/etc/prometheus/targets/{{ target.job }}/{{ target.instance }}.yml" + state: absent + notify: reload prometheus - name: generate rules files for all jobs - loop: "{{ (prometheus_server_jobs_generic + prometheus_server_jobs_special) | union(['prometheus']) }}" + loop: "{{ prometheus_server_jobs | union(['prometheus']) }}" template: src: rules.yml.j2 dest: "/etc/prometheus/rules/{{ item }}.yml" diff --git a/roles/monitoring/prometheus/server/templates/job-snippets/blackbox-https.j2 b/roles/monitoring/prometheus/server/templates/job-snippets/blackbox-https.j2 index 0a6d2dfa..98a64121 100644 --- a/roles/monitoring/prometheus/server/templates/job-snippets/blackbox-https.j2 +++ b/roles/monitoring/prometheus/server/templates/job-snippets/blackbox-https.j2 @@ -11,4 +11,4 @@ key_file: /etc/ssl/prometheus/server/scrape-key.pem file_sd_configs: - files: - - "/etc/prometheus/jobs/{{ job }}/*.yml" + - "/etc/prometheus/targets/{{ job }}/*.yml" diff --git a/roles/monitoring/prometheus/server/templates/job-snippets/blackbox-ping.j2 b/roles/monitoring/prometheus/server/templates/job-snippets/blackbox-ping.j2 index 7f4f12df..736ffec1 100644 --- a/roles/monitoring/prometheus/server/templates/job-snippets/blackbox-ping.j2 +++ b/roles/monitoring/prometheus/server/templates/job-snippets/blackbox-ping.j2 @@ -11,4 +11,4 @@ key_file: /etc/ssl/prometheus/server/scrape-key.pem file_sd_configs: - files: - - "/etc/prometheus/jobs/{{ job }}/*.yml" + - "/etc/prometheus/targets/{{ job }}/*.yml" diff --git a/roles/monitoring/prometheus/server/templates/job-snippets/blackbox-ssh.j2 b/roles/monitoring/prometheus/server/templates/job-snippets/blackbox-ssh.j2 index 18381e32..166f37ad 100644 --- a/roles/monitoring/prometheus/server/templates/job-snippets/blackbox-ssh.j2 +++ b/roles/monitoring/prometheus/server/templates/job-snippets/blackbox-ssh.j2 @@ -11,4 +11,4 @@ key_file: /etc/ssl/prometheus/server/scrape-key.pem file_sd_configs: - files: - - "/etc/prometheus/jobs/{{ job }}/*.yml" + - "/etc/prometheus/targets/{{ job }}/*.yml" diff --git a/roles/monitoring/prometheus/server/templates/job-snippets/generic.j2 b/roles/monitoring/prometheus/server/templates/job-snippets/generic.j2 index 87992eeb..b155c5f7 100644 --- a/roles/monitoring/prometheus/server/templates/job-snippets/generic.j2 +++ b/roles/monitoring/prometheus/server/templates/job-snippets/generic.j2 @@ -10,4 +10,4 @@ key_file: /etc/ssl/prometheus/server/scrape-key.pem file_sd_configs: - files: - - "/etc/prometheus/jobs/{{ job }}/*.yml" + - "/etc/prometheus/targets/{{ job }}/*.yml" diff --git a/roles/monitoring/prometheus/server/templates/job-snippets/nut-ups.j2 b/roles/monitoring/prometheus/server/templates/job-snippets/nut-ups.j2 new file mode 100644 index 00000000..3a2c5c62 --- /dev/null +++ b/roles/monitoring/prometheus/server/templates/job-snippets/nut-ups.j2 @@ -0,0 +1,13 @@ + - job_name: '{{ job }}' + metrics_path: /proxy + params: + module: + - nut + scheme: https + tls_config: + ca_file: /etc/ssl/prometheus/ca-crt.pem + cert_file: /etc/ssl/prometheus/server/scrape-crt.pem + key_file: /etc/ssl/prometheus/server/scrape-key.pem + file_sd_configs: + - files: + - "/etc/prometheus/targets/{{ job }}/*.yml" diff --git a/roles/monitoring/prometheus/server/templates/job-snippets/openwrt.j2 b/roles/monitoring/prometheus/server/templates/job-snippets/openwrt.j2 index 493a4fdb..e93f8be7 100644 --- a/roles/monitoring/prometheus/server/templates/job-snippets/openwrt.j2 +++ b/roles/monitoring/prometheus/server/templates/job-snippets/openwrt.j2 @@ -2,4 +2,4 @@ scheme: http file_sd_configs: - files: - - "/etc/prometheus/jobs/{{ job }}/*.yml" + - "/etc/prometheus/targets/{{ job }}/*.yml" diff --git a/roles/monitoring/prometheus/server/templates/prometheus.yml.j2 b/roles/monitoring/prometheus/server/templates/prometheus.yml.j2 index 4cfcc498..3286bb82 100644 --- a/roles/monitoring/prometheus/server/templates/prometheus.yml.j2 +++ b/roles/monitoring/prometheus/server/templates/prometheus.yml.j2 @@ -35,7 +35,7 @@ scrape_configs: static_configs: - targets: ['{{ prometheus_server_alertmanager.url }}'] {% endif %} -{% for job in (prometheus_server_jobs_generic + prometheus_server_jobs_special) %} +{% for job in (prometheus_server_jobs) %} {% include 'job-snippets/' + (lookup('first_found', {'paths': ['templates/job-snippets'], 'files': [job + '.j2', 'generic.j2']}) | basename) %}{{ '' }} {% endfor %} diff --git a/roles/monitoring/prometheus/server/templates/targets/generic.yml.j2 b/roles/monitoring/prometheus/server/templates/targets/generic.yml.j2 index e83b6bf4..45af10c0 100644 --- a/roles/monitoring/prometheus/server/templates/targets/generic.yml.j2 +++ b/roles/monitoring/prometheus/server/templates/targets/generic.yml.j2 @@ -1,3 +1,3 @@ -- targets: [ "{{ hostvars[target].prometheus_scrape_endpoint }}" ] +- targets: [ "{{ hostvars[target.instance].prometheus_scrape_endpoint }}" ] labels: - instance: "{{ target }}" + instance: "{{ target.instance }}" diff --git a/roles/monitoring/prometheus/server/templates/targets/nut.yml.j2 b/roles/monitoring/prometheus/server/templates/targets/nut-ups.yml.j2 index da3de3d7..da3de3d7 100644 --- a/roles/monitoring/prometheus/server/templates/targets/nut.yml.j2 +++ b/roles/monitoring/prometheus/server/templates/targets/nut-ups.yml.j2 |