prometheus: add support special jobs (WIP)

7 files changed, 65 insertions, 16 deletions

diff --git a/inventory/group_vars/chaos-at-home-ups/vars.yml b/inventory/group_vars/chaos-at-home-ups/vars.yml
index 3bdcd7ea..cc7ae585 100644
--- a/inventory/group_vars/chaos-at-home-ups/vars.yml
+++ b/inventory/group_vars/chaos-at-home-ups/vars.yml
@@ -10,3 +10,9 @@ ups_query_sources:
 prometheus_scrape_endpoint: "{{ network_mgmt_zone.prefix | ipaddr(network_mgmt_zone.offsets[inventory_hostname]) | ipaddr('address') }}:9100"
 prometheus_exporters_default:
   - openwrt
+
+prometheus_special_job_nut:
+  exporter_hostname: ch-mon
+  instance: "ups-{{ ups_name }}"
+  ups: "{{ ups_name }}"
+  server: "{{ network_mgmt_zone.prefix | ipaddr(network_mgmt_zone.offsets[inventory_hostname]) | ipaddr('address') }}"
diff --git a/roles/monitoring/prometheus/server/filter_plugins/prometheus.py b/roles/monitoring/prometheus/server/filter_plugins/prometheus.py
index 81cfae70..6e18481c 100644
--- a/roles/monitoring/prometheus/server/filter_plugins/prometheus.py
+++ b/roles/monitoring/prometheus/server/filter_plugins/prometheus.py
@@ -6,23 +6,37 @@ from functools import partial
 from ansible import errors
 
 
-def prometheus_job_targets(hostvars, jobs, targets):
+def prometheus_generic_job_targets(hostvars, jobs, targets):
     try:
         result = []
         for job in jobs:
             for target in targets:
                 enabled = job in hostvars[target]['prometheus_exporters_default'] or job in hostvars[target]['prometheus_exporters_extra']
-                result.append({'job': job, 'target': target, 'enabled': enabled})
+                result.append({'job': job, 'instance': target, 'enabled': enabled})
         return result
     except Exception as e:
-        raise errors.AnsibleFilterError("prometheus_job_targets(): %s" % str(e))
+        raise errors.AnsibleFilterError("prometheus_generic_job_targets(): %s" % str(e))
+
+
+def prometheus_special_job_targets(hostvars, jobs, targets):
+    try:
+        result = []
+        for job in jobs:
+            for target in targets:
+                config_varname = 'prometheus_special_job_' + job
+                if config_varname in hostvars[target]:
+                    result.append({'job': job, 'instance': hostvars[target][config_varname]['instance'], 'config': hostvars[target][config_varname]})
+        return result
+    except Exception as e:
+        raise errors.AnsibleFilterError("prometheus_special_job_targets(): %s" % str(e))
 
 
 class FilterModule(object):
 
     ''' prometheus filters '''
     filter_map = {
-        'prometheus_job_targets': prometheus_job_targets,
+        'prometheus_generic_job_targets': prometheus_generic_job_targets,
+        'prometheus_special_job_targets': prometheus_special_job_targets,
     }
 
     def filters(self):
diff --git a/roles/monitoring/prometheus/server/tasks/main.yml b/roles/monitoring/prometheus/server/tasks/main.yml
index 1e25eb06..d0ccd8af 100644
--- a/roles/monitoring/prometheus/server/tasks/main.yml
+++ b/roles/monitoring/prometheus/server/tasks/main.yml
@@ -52,23 +52,32 @@
 
 - name: generate generic targets config
   loop: "{{ prometheus_zone_targets }}"
-  copy:
-    content: |
-      - targets: [ "{{ hostvars[item].prometheus_scrape_endpoint }}" ]
-        labels:
-          instance: "{{ item }}"
-    dest: "/etc/prometheus/targets/{{ item }}.yml"
+  loop_control:
+    loop_var: target
+  template:
+    src: targets/generic.yml.j2
+    dest: "/etc/prometheus/targets/{{ target }}.yml"
+  notify: reload prometheus
 
 - name: enable targets for generic jobs
-  loop: "{{ hostvars | prometheus_job_targets(prometheus_server_jobs_generic, prometheus_zone_targets) }}"
+  loop: "{{ hostvars | prometheus_generic_job_targets(prometheus_server_jobs_generic, prometheus_zone_targets) }}"
   loop_control:
-    label: "{{ item.job }} -> {{ item.target }}"
+    label: "{{ item.job }} -> {{ item.instance }}"
   file:
-    src: "{{ item.enabled | ternary('/etc/prometheus/targets/' + item.target + '.yml', omit) }}"
-    path: "/etc/prometheus/jobs/{{ item.job }}/{{ item.target }}.yml"
+    src: "{{ item.enabled | ternary('/etc/prometheus/targets/' + item.instance + '.yml', omit) }}"
+    path: "/etc/prometheus/jobs/{{ item.job }}/{{ item.instance }}.yml"
     state: "{{ item.enabled | ternary('link', 'absent') }}"
+  notify: reload prometheus
 
-## TODO: generate special target configs
+- name: enable targets for special jobs
+  loop: "{{ hostvars | prometheus_special_job_targets(prometheus_server_jobs_special, prometheus_zone_targets) }}"
+  loop_control:
+    loop_var: target
+    label: "{{ target.job }} -> {{ target.instance }}"
+  template:
+    src: "targets/{{ target.job }}.yml.j2"
+    dest: "/etc/prometheus/jobs/{{ target.job }}/{{ target.instance }}.yml"
+  notify: reload prometheus
 
 - name: generate rules files for all jobs
   loop: "{{ (prometheus_server_jobs_generic + prometheus_server_jobs_special) | union(['prometheus']) }}"
diff --git a/roles/monitoring/prometheus/server/templates/job-snippets/default.j2 b/roles/monitoring/prometheus/server/templates/job-snippets/generic.j2
index 87992eeb..87992eeb 100644
--- a/roles/monitoring/prometheus/server/templates/job-snippets/default.j2
+++ b/roles/monitoring/prometheus/server/templates/job-snippets/generic.j2
diff --git a/roles/monitoring/prometheus/server/templates/prometheus.yml.j2 b/roles/monitoring/prometheus/server/templates/prometheus.yml.j2
index 2b9a08e4..4cfcc498 100644
--- a/roles/monitoring/prometheus/server/templates/prometheus.yml.j2
+++ b/roles/monitoring/prometheus/server/templates/prometheus.yml.j2
@@ -37,7 +37,7 @@ scrape_configs:
 {% endif %}
 {% for job in (prometheus_server_jobs_generic + prometheus_server_jobs_special) %}
 
-{%   include 'job-snippets/' + (lookup('first_found', {'paths': ['templates/job-snippets'], 'files': [job + '.j2', 'default.j2']}) | basename) %}{{ '' }}
+{%   include 'job-snippets/' + (lookup('first_found', {'paths': ['templates/job-snippets'], 'files': [job + '.j2', 'generic.j2']}) | basename) %}{{ '' }}
 {% endfor %}
 {% if prometheus_server_jobs_extra is defined %}
 
diff --git a/roles/monitoring/prometheus/server/templates/targets/generic.yml.j2 b/roles/monitoring/prometheus/server/templates/targets/generic.yml.j2
new file mode 100644
index 00000000..e83b6bf4
--- /dev/null
+++ b/roles/monitoring/prometheus/server/templates/targets/generic.yml.j2
@@ -0,0 +1,3 @@
+- targets: [ "{{ hostvars[target].prometheus_scrape_endpoint }}" ]
+  labels:
+    instance: "{{ target }}"
diff --git a/roles/monitoring/prometheus/server/templates/targets/nut.yml.j2 b/roles/monitoring/prometheus/server/templates/targets/nut.yml.j2
new file mode 100644
index 00000000..da3de3d7
--- /dev/null
+++ b/roles/monitoring/prometheus/server/templates/targets/nut.yml.j2
@@ -0,0 +1,17 @@
+- targets: [ "{{ hostvars[target.config.exporter_hostname].prometheus_scrape_endpoint }}" ]
+  labels:
+    instance: "{{ target.instance }}"
+    __param_ups: {{ target.config.ups }}
+    __param_server: {{ target.config.server | default('127.0.0.1') }}
+{% if 'username' in target.config %}
+    __param_username: {{ target.config.username }}
+{% endif  %}
+{% if 'password' in target.config %}
+    __param_password: {{ target.config.password }}
+{% endif  %}
+{% if 'variables' in target.config %}
+    __param_variables: {{ target.config.variables }}
+{% endif  %}
+{% if 'statuses' in target.config %}
+    __param_statuses: {{ target.config.statuses }}
+{% endif  %}