add whawty-nginx-sso to prometheus monitoring

13 files changed, 105 insertions, 0 deletions

diff --git a/chaos-at-home/ch-http-proxy.yml b/chaos-at-home/ch-http-proxy.yml
index aa2ad3ef..cee4474e 100644
--- a/chaos-at-home/ch-http-proxy.yml
+++ b/chaos-at-home/ch-http-proxy.yml
@@ -6,11 +6,13 @@
   - role: core/base
   - role: core/sshd/base
   - role: core/zsh
+  - role: core/ntp
 
 - name: Payload Setup
   hosts: ch-http-proxy
   roles:
   - role: apt-repo/spreadspace
+  - role: monitoring/prometheus/exporter
   - role: x509/acmetool/base
   - role: nginx/base
   - role: nginx/auth/whawty-sso/base
diff --git a/inventory/group_vars/promzone-chaos-at-home/vars.yml b/inventory/group_vars/promzone-chaos-at-home/vars.yml
index 430fbdd5..47ee79aa 100644
--- a/inventory/group_vars/promzone-chaos-at-home/vars.yml
+++ b/inventory/group_vars/promzone-chaos-at-home/vars.yml
@@ -33,6 +33,7 @@ prometheus_server_jobs:
   - smokeping
   - bind
   - standalone-kubelet
+  - whawty-nginx-sso
 
 prometheus_zone_name: chaos@home
 
diff --git a/inventory/host_vars/ch-http-proxy.yml b/inventory/host_vars/ch-http-proxy.yml
index d26259b9..fc17187e 100644
--- a/inventory/host_vars/ch-http-proxy.yml
+++ b/inventory/host_vars/ch-http-proxy.yml
@@ -33,9 +33,27 @@ network:
   - *_network_primary_
 
 
+ntp_variant: systemd-timesyncd
+
+
 acme_directory_server: "{{ acme_directory_server_le_live_v2 }}"
 
 
+spreadspace_apt_repo_components:
+  - main
+  - prometheus
+
+prometheus_job_multitarget_blackbox__probe:
+  ch-mon:
+  - instance: "ssh-{{ inventory_hostname }}"
+    target: "{{ network_zones.svc.prefix | ansible.utils.ipaddr(network_zones.svc.offsets[inventory_hostname]) | ansible.utils.ipaddr('address') }}:{{ ansible_port | default(22) }}"
+    module: ssh_banner
+  - instance: "https-login.chaos-at-home.org"
+    target: "https://{{ network_services.http.addr }}/login"
+    module: "http_tls_2xx"
+    hostname: "login.chaos-at-home.org"
+
+
 whawty_nginx_sso_backends:
   chaos-at-home:
     port: 1234
@@ -71,5 +89,12 @@ whawty_nginx_sso_logins:
           title: "chaoSSO login"
         revocations:
           tokens: "{{ vault_whawty_nginx_sso_sync_tokens['chaos-at-home'] | dict2items | map(attribute='value') }}"
+      prometheus:
+        listen: 127.0.0.1:1235
 
 whawty_nginx_sso_login_static_credentials__chaos-at-home: "{{ vault_whawty_nginx_sso_login_static_credentials['chaos-at-home'] }}"
+
+prometheus_job_multitarget_whawty_nginx_sso:
+  ch-mon:
+  - instance: "whawty-nginx-sso-{{ inventory_hostname }}-chaos-at-home"
+    instance_name: chaos-at-home
diff --git a/inventory/host_vars/ch-mon.yml b/inventory/host_vars/ch-mon.yml
index d7cb9aaa..63bb7cb6 100644
--- a/inventory/host_vars/ch-mon.yml
+++ b/inventory/host_vars/ch-mon.yml
@@ -93,6 +93,12 @@ whawty_nginx_sso_auths:
               server-name: "login.chaos-at-home.org"
       web:
         listen: 127.0.0.1:1234
+      prometheus: {}
+
+prometheus_job_multitarget_whawty_nginx_sso:
+  ch-mon:
+  - instance: "whawty-nginx-sso-{{ inventory_hostname }}-chaos-at-home"
+    instance_name: chaos-at-home
 
 
 prometheus_server_storage:
diff --git a/inventory/hosts.ini b/inventory/hosts.ini
index 0db902a0..be63066b 100644
--- a/inventory/hosts.ini
+++ b/inventory/hosts.ini
@@ -480,6 +480,7 @@ ch-atlas
 ch-pan
 ch-mimas
 ch-iot
+ch-http-proxy
 [promzone-chaos-at-home:children]
 chaos-at-home-ap
 chaos-at-home-ups
diff --git a/roles/monitoring/prometheus/exporter/register/templates/blackbox/probe.yml.j2 b/roles/monitoring/prometheus/exporter/register/templates/blackbox/probe.yml.j2
index 3ecf129a..5f98e3a8 100644
--- a/roles/monitoring/prometheus/exporter/register/templates/blackbox/probe.yml.j2
+++ b/roles/monitoring/prometheus/exporter/register/templates/blackbox/probe.yml.j2
@@ -3,6 +3,9 @@
     instance: '{{ target.instance }}'
     __param_target: '{{ target.config.target }}'
     __param_module: '{{ target.config.module }}'
+{% if 'hostname' in target.config %}
+    __param_hostname: '{{ target.config.hostname }}'
+{% endif %}
 {% for name, value in prometheus_target_labels.items() %}
     {{ name }}: '{{ value }}'
 {% endfor %}
diff --git a/roles/monitoring/prometheus/exporter/register/templates/whawty-nginx-sso.yml.j2 b/roles/monitoring/prometheus/exporter/register/templates/whawty-nginx-sso.yml.j2
new file mode 100644
index 00000000..74ada64e
--- /dev/null
+++ b/roles/monitoring/prometheus/exporter/register/templates/whawty-nginx-sso.yml.j2
@@ -0,0 +1,7 @@
+- targets: [ '{{ (target.exporter_hostname == prometheus_server) | ternary('127.0.0.1:9999', hostvars[target.exporter_hostname].prometheus_scrape_endpoint) }}' ]
+  labels:
+    instance: '{{ target.instance }}'
+    __metrics_path__: '/whawty-nginx-sso/{{ target.config.instance_name }}'
+{% for name, value in prometheus_target_labels.items() %}
+    {{ name }}: '{{ value }}'
+{% endfor %}
diff --git a/roles/monitoring/prometheus/server/defaults/main/main.yml b/roles/monitoring/prometheus/server/defaults/main/main.yml
index 7a9adde4..7f78d5aa 100644
--- a/roles/monitoring/prometheus/server/defaults/main/main.yml
+++ b/roles/monitoring/prometheus/server/defaults/main/main.yml
@@ -39,6 +39,7 @@ prometheus_server_rules:
   modbus: "{{ prometheus_server_rules_modbus + prometheus_server_rules_modbus_extra }}"
   modbus/probe: "{{ prometheus_server_rules_modbus__probe + prometheus_server_rules_modbus__probe_extra }}"
   nftables: "{{ prometheus_server_rules_nftables + prometheus_server_rules_nftables_extra }}"
+  whawty-nginx-sso: "{{ prometheus_server_rules_whawty_nginx_sso + prometheus_server_rules_whawty_nginx_sso_extra }}"
 
 # prometheus_server_alertmanager:
 #   url: "127.0.0.1:9093"
diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_whawty-nginx-sso.yml b/roles/monitoring/prometheus/server/defaults/main/rules_whawty-nginx-sso.yml
new file mode 100644
index 00000000..215cbf4b
--- /dev/null
+++ b/roles/monitoring/prometheus/server/defaults/main/rules_whawty-nginx-sso.yml
@@ -0,0 +1,3 @@
+---
+prometheus_server_rules_whawty_nginx_sso_extra: []
+prometheus_server_rules_whawty_nginx_sso: []
diff --git a/roles/nginx/auth/whawty-sso/auth/handlers/main.yml b/roles/nginx/auth/whawty-sso/auth/handlers/main.yml
index fad676ce..415a976a 100644
--- a/roles/nginx/auth/whawty-sso/auth/handlers/main.yml
+++ b/roles/nginx/auth/whawty-sso/auth/handlers/main.yml
@@ -4,3 +4,8 @@
   service:
     name: "whawty-nginx-sso@{{ item }}.service"
     state: restarted
+
+- name: reload nginx
+  service:
+    name: nginx
+    state: reloaded
diff --git a/roles/nginx/auth/whawty-sso/auth/tasks/main.yml b/roles/nginx/auth/whawty-sso/auth/tasks/main.yml
index 5ae64b9b..a2bba813 100644
--- a/roles/nginx/auth/whawty-sso/auth/tasks/main.yml
+++ b/roles/nginx/auth/whawty-sso/auth/tasks/main.yml
@@ -13,6 +13,29 @@
     state: directory
     mode: 0700
 
+- name: make sure prometheus directories exist
+  when: "(whawty_nginx_sso_auths | dict2items | selectattr('value.config.prometheus', 'defined') | length) > 0"
+  file:
+    path: /etc/prometheus/exporter
+    state: directory
+
+- name: make sure prometheus directories exist
+  loop: "{{ whawty_nginx_sso_auths | dict2items | selectattr('value.config.prometheus', 'defined') }}"
+  loop_control:
+    label: "{{ item.key }}"
+  copy:
+    content: |
+      {% if 'listen' in item.value.config.prometheus %}
+      {%   set listen = item.value.config.prometheus.listen %}
+      {% else %}
+      {%   set listen = item.value.config.web.listen %}
+      {% endif %}
+      location = /whawty-nginx-sso/{{ item.key }} {
+          proxy_pass http://127.0.0.1:{{ listen | split(':') | last }}{{ item.value.config.prometheus.path | default('/metrics') }};
+      }
+    dest: "/etc/prometheus/exporter/whawty-nginx-sso-{{ item.key }}.locations"
+  notify: reload nginx
+
 - name: generate configuration file
   loop: "{{ whawty_nginx_sso_auths | dict2items }}"
   loop_control:
diff --git a/roles/nginx/auth/whawty-sso/login/handlers/main.yml b/roles/nginx/auth/whawty-sso/login/handlers/main.yml
index f4bbf308..dea155d1 100644
--- a/roles/nginx/auth/whawty-sso/login/handlers/main.yml
+++ b/roles/nginx/auth/whawty-sso/login/handlers/main.yml
@@ -4,3 +4,8 @@
   service:
     name: "whawty-nginx-sso@{{ item }}.service"
     state: restarted
+
+- name: reload nginx
+  service:
+    name: nginx
+    state: reloaded
diff --git a/roles/nginx/auth/whawty-sso/login/tasks/main.yml b/roles/nginx/auth/whawty-sso/login/tasks/main.yml
index e2267238..675a3ffa 100644
--- a/roles/nginx/auth/whawty-sso/login/tasks/main.yml
+++ b/roles/nginx/auth/whawty-sso/login/tasks/main.yml
@@ -25,6 +25,29 @@
     state: directory
     mode: 0700
 
+- name: make sure prometheus directories exist
+  when: "(whawty_nginx_sso_logins | dict2items | selectattr('value.config.prometheus', 'defined') | length) > 0"
+  file:
+    path: /etc/prometheus/exporter
+    state: directory
+
+- name: make sure prometheus directories exist
+  loop: "{{ whawty_nginx_sso_logins | dict2items | selectattr('value.config.prometheus', 'defined') }}"
+  loop_control:
+    label: "{{ item.key }}"
+  copy:
+    content: |
+      {% if 'listen' in item.value.config.prometheus %}
+      {%   set listen = item.value.config.prometheus.listen %}
+      {% else %}
+      {%   set listen = item.value.config.web.listen %}
+      {% endif %}
+      location = /whawty-nginx-sso/{{ item.key }} {
+          proxy_pass http://127.0.0.1:{{ listen | split(':') | last }}{{ item.value.config.prometheus.path | default('/metrics') }};
+      }
+    dest: "/etc/prometheus/exporter/whawty-nginx-sso-{{ item.key }}.locations"
+  notify: reload nginx
+
 
 - name: generate configuration file
   loop: "{{ whawty_nginx_sso_logins | dict2items }}"