From 5984697d98b5c953101aa8eef6c69db307d4a563 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Sat, 28 Aug 2021 18:35:24 +0200 Subject: prometheus: add support special jobs (WIP) --- inventory/group_vars/chaos-at-home-ups/vars.yml | 6 +++++ .../prometheus/server/filter_plugins/prometheus.py | 22 ++++++++++++--- roles/monitoring/prometheus/server/tasks/main.yml | 31 ++++++++++++++-------- .../server/templates/job-snippets/default.j2 | 13 --------- .../server/templates/job-snippets/generic.j2 | 13 +++++++++ .../prometheus/server/templates/prometheus.yml.j2 | 2 +- .../server/templates/targets/generic.yml.j2 | 3 +++ .../prometheus/server/templates/targets/nut.yml.j2 | 17 ++++++++++++ 8 files changed, 78 insertions(+), 29 deletions(-) delete mode 100644 roles/monitoring/prometheus/server/templates/job-snippets/default.j2 create mode 100644 roles/monitoring/prometheus/server/templates/job-snippets/generic.j2 create mode 100644 roles/monitoring/prometheus/server/templates/targets/generic.yml.j2 create mode 100644 roles/monitoring/prometheus/server/templates/targets/nut.yml.j2 diff --git a/inventory/group_vars/chaos-at-home-ups/vars.yml b/inventory/group_vars/chaos-at-home-ups/vars.yml index 3bdcd7ea..cc7ae585 100644 --- a/inventory/group_vars/chaos-at-home-ups/vars.yml +++ b/inventory/group_vars/chaos-at-home-ups/vars.yml @@ -10,3 +10,9 @@ ups_query_sources: prometheus_scrape_endpoint: "{{ network_mgmt_zone.prefix | ipaddr(network_mgmt_zone.offsets[inventory_hostname]) | ipaddr('address') }}:9100" prometheus_exporters_default: - openwrt + +prometheus_special_job_nut: + exporter_hostname: ch-mon + instance: "ups-{{ ups_name }}" + ups: "{{ ups_name }}" + server: "{{ network_mgmt_zone.prefix | ipaddr(network_mgmt_zone.offsets[inventory_hostname]) | ipaddr('address') }}" diff --git a/roles/monitoring/prometheus/server/filter_plugins/prometheus.py b/roles/monitoring/prometheus/server/filter_plugins/prometheus.py index 81cfae70..6e18481c 100644 --- a/roles/monitoring/prometheus/server/filter_plugins/prometheus.py +++ b/roles/monitoring/prometheus/server/filter_plugins/prometheus.py @@ -6,23 +6,37 @@ from functools import partial from ansible import errors -def prometheus_job_targets(hostvars, jobs, targets): +def prometheus_generic_job_targets(hostvars, jobs, targets): try: result = [] for job in jobs: for target in targets: enabled = job in hostvars[target]['prometheus_exporters_default'] or job in hostvars[target]['prometheus_exporters_extra'] - result.append({'job': job, 'target': target, 'enabled': enabled}) + result.append({'job': job, 'instance': target, 'enabled': enabled}) return result except Exception as e: - raise errors.AnsibleFilterError("prometheus_job_targets(): %s" % str(e)) + raise errors.AnsibleFilterError("prometheus_generic_job_targets(): %s" % str(e)) + + +def prometheus_special_job_targets(hostvars, jobs, targets): + try: + result = [] + for job in jobs: + for target in targets: + config_varname = 'prometheus_special_job_' + job + if config_varname in hostvars[target]: + result.append({'job': job, 'instance': hostvars[target][config_varname]['instance'], 'config': hostvars[target][config_varname]}) + return result + except Exception as e: + raise errors.AnsibleFilterError("prometheus_special_job_targets(): %s" % str(e)) class FilterModule(object): ''' prometheus filters ''' filter_map = { - 'prometheus_job_targets': prometheus_job_targets, + 'prometheus_generic_job_targets': prometheus_generic_job_targets, + 'prometheus_special_job_targets': prometheus_special_job_targets, } def filters(self): diff --git a/roles/monitoring/prometheus/server/tasks/main.yml b/roles/monitoring/prometheus/server/tasks/main.yml index 1e25eb06..d0ccd8af 100644 --- a/roles/monitoring/prometheus/server/tasks/main.yml +++ b/roles/monitoring/prometheus/server/tasks/main.yml @@ -52,23 +52,32 @@ - name: generate generic targets config loop: "{{ prometheus_zone_targets }}" - copy: - content: | - - targets: [ "{{ hostvars[item].prometheus_scrape_endpoint }}" ] - labels: - instance: "{{ item }}" - dest: "/etc/prometheus/targets/{{ item }}.yml" + loop_control: + loop_var: target + template: + src: targets/generic.yml.j2 + dest: "/etc/prometheus/targets/{{ target }}.yml" + notify: reload prometheus - name: enable targets for generic jobs - loop: "{{ hostvars | prometheus_job_targets(prometheus_server_jobs_generic, prometheus_zone_targets) }}" + loop: "{{ hostvars | prometheus_generic_job_targets(prometheus_server_jobs_generic, prometheus_zone_targets) }}" loop_control: - label: "{{ item.job }} -> {{ item.target }}" + label: "{{ item.job }} -> {{ item.instance }}" file: - src: "{{ item.enabled | ternary('/etc/prometheus/targets/' + item.target + '.yml', omit) }}" - path: "/etc/prometheus/jobs/{{ item.job }}/{{ item.target }}.yml" + src: "{{ item.enabled | ternary('/etc/prometheus/targets/' + item.instance + '.yml', omit) }}" + path: "/etc/prometheus/jobs/{{ item.job }}/{{ item.instance }}.yml" state: "{{ item.enabled | ternary('link', 'absent') }}" + notify: reload prometheus -## TODO: generate special target configs +- name: enable targets for special jobs + loop: "{{ hostvars | prometheus_special_job_targets(prometheus_server_jobs_special, prometheus_zone_targets) }}" + loop_control: + loop_var: target + label: "{{ target.job }} -> {{ target.instance }}" + template: + src: "targets/{{ target.job }}.yml.j2" + dest: "/etc/prometheus/jobs/{{ target.job }}/{{ target.instance }}.yml" + notify: reload prometheus - name: generate rules files for all jobs loop: "{{ (prometheus_server_jobs_generic + prometheus_server_jobs_special) | union(['prometheus']) }}" diff --git a/roles/monitoring/prometheus/server/templates/job-snippets/default.j2 b/roles/monitoring/prometheus/server/templates/job-snippets/default.j2 deleted file mode 100644 index 87992eeb..00000000 --- a/roles/monitoring/prometheus/server/templates/job-snippets/default.j2 +++ /dev/null @@ -1,13 +0,0 @@ - - job_name: '{{ job }}' - metrics_path: /proxy - params: - module: - - {{ job }} - scheme: https - tls_config: - ca_file: /etc/ssl/prometheus/ca-crt.pem - cert_file: /etc/ssl/prometheus/server/scrape-crt.pem - key_file: /etc/ssl/prometheus/server/scrape-key.pem - file_sd_configs: - - files: - - "/etc/prometheus/jobs/{{ job }}/*.yml" diff --git a/roles/monitoring/prometheus/server/templates/job-snippets/generic.j2 b/roles/monitoring/prometheus/server/templates/job-snippets/generic.j2 new file mode 100644 index 00000000..87992eeb --- /dev/null +++ b/roles/monitoring/prometheus/server/templates/job-snippets/generic.j2 @@ -0,0 +1,13 @@ + - job_name: '{{ job }}' + metrics_path: /proxy + params: + module: + - {{ job }} + scheme: https + tls_config: + ca_file: /etc/ssl/prometheus/ca-crt.pem + cert_file: /etc/ssl/prometheus/server/scrape-crt.pem + key_file: /etc/ssl/prometheus/server/scrape-key.pem + file_sd_configs: + - files: + - "/etc/prometheus/jobs/{{ job }}/*.yml" diff --git a/roles/monitoring/prometheus/server/templates/prometheus.yml.j2 b/roles/monitoring/prometheus/server/templates/prometheus.yml.j2 index 2b9a08e4..4cfcc498 100644 --- a/roles/monitoring/prometheus/server/templates/prometheus.yml.j2 +++ b/roles/monitoring/prometheus/server/templates/prometheus.yml.j2 @@ -37,7 +37,7 @@ scrape_configs: {% endif %} {% for job in (prometheus_server_jobs_generic + prometheus_server_jobs_special) %} -{% include 'job-snippets/' + (lookup('first_found', {'paths': ['templates/job-snippets'], 'files': [job + '.j2', 'default.j2']}) | basename) %}{{ '' }} +{% include 'job-snippets/' + (lookup('first_found', {'paths': ['templates/job-snippets'], 'files': [job + '.j2', 'generic.j2']}) | basename) %}{{ '' }} {% endfor %} {% if prometheus_server_jobs_extra is defined %} diff --git a/roles/monitoring/prometheus/server/templates/targets/generic.yml.j2 b/roles/monitoring/prometheus/server/templates/targets/generic.yml.j2 new file mode 100644 index 00000000..e83b6bf4 --- /dev/null +++ b/roles/monitoring/prometheus/server/templates/targets/generic.yml.j2 @@ -0,0 +1,3 @@ +- targets: [ "{{ hostvars[target].prometheus_scrape_endpoint }}" ] + labels: + instance: "{{ target }}" diff --git a/roles/monitoring/prometheus/server/templates/targets/nut.yml.j2 b/roles/monitoring/prometheus/server/templates/targets/nut.yml.j2 new file mode 100644 index 00000000..da3de3d7 --- /dev/null +++ b/roles/monitoring/prometheus/server/templates/targets/nut.yml.j2 @@ -0,0 +1,17 @@ +- targets: [ "{{ hostvars[target.config.exporter_hostname].prometheus_scrape_endpoint }}" ] + labels: + instance: "{{ target.instance }}" + __param_ups: {{ target.config.ups }} + __param_server: {{ target.config.server | default('127.0.0.1') }} +{% if 'username' in target.config %} + __param_username: {{ target.config.username }} +{% endif %} +{% if 'password' in target.config %} + __param_password: {{ target.config.password }} +{% endif %} +{% if 'variables' in target.config %} + __param_variables: {{ target.config.variables }} +{% endif %} +{% if 'statuses' in target.config %} + __param_statuses: {{ target.config.statuses }} +{% endif %} -- cgit v1.2.3