From 1d94d5f6e3cf85f1e0be0eb6b45cc1a5f1f27b7f Mon Sep 17 00:00:00 2001
From: Christian Pointner <equinox@spreadspace.org>
Date: Sun, 28 Jan 2024 13:37:07 +0100
Subject: whawty/auth/store: add prometheus metrics for sync

---
 inventory/group_vars/promzone-chaos-at-home/vars.yml | 20 ++++++++++++++++++++
 inventory/host_vars/ch-http-proxy.yml                |  1 +
 inventory/host_vars/ch-pan.yml                       |  1 +
 roles/whawty/auth/store/defaults/main.yml            |  1 +
 roles/whawty/auth/store/tasks/sync-client.yml        | 10 +++++++++-
 roles/whawty/auth/store/templates/systemd.service.j2 |  3 ++-
 6 files changed, 34 insertions(+), 2 deletions(-)

diff --git a/inventory/group_vars/promzone-chaos-at-home/vars.yml b/inventory/group_vars/promzone-chaos-at-home/vars.yml
index 8c3b7945..1e3c2ada 100644
--- a/inventory/group_vars/promzone-chaos-at-home/vars.yml
+++ b/inventory/group_vars/promzone-chaos-at-home/vars.yml
@@ -41,3 +41,23 @@ prometheus_zone_name: chaos@home
 
 ## TODO:
 ## potential extra alert rule: (bind_zone_serial{instance="ch-mimas"} == bool on(job, view, zone_name) bind_zone_serial{instance="ch-pan"}) != 1
+
+
+prometheus_server_rules_whawty_nginx_sso_extra:
+  - alert: WhawtyAuthStoreSyncTooLongAgo
+    expr: time() - whawty_auth_store_sync_run > 3600
+    for: 0m
+    labels:
+      severity: warning
+    annotations:
+      summary: The last whawty-auth store sync was too long ago (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "The last whawty-auth store sync of {{ '{{' }} $labels.name {{ '}}' }} on {{ '{{' }} $labels.instance {{ '}}' }} ran more then an hours ago.\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: WhawtyAuthStoreSyncFailed
+    expr: whawty_auth_store_sync_exit_code != 0
+    for: 0m
+    labels:
+      severity: warning
+    annotations:
+      summary: The last whawty-auth sync failed (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "The last whawty-auth store sync of {{ '{{' }} $labels.name {{ '}}' }} on {{ '{{' }} $labels.instance {{ '}}' }} has failed.\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
diff --git a/inventory/host_vars/ch-http-proxy.yml b/inventory/host_vars/ch-http-proxy.yml
index 46e63c1d..5be067ec 100644
--- a/inventory/host_vars/ch-http-proxy.yml
+++ b/inventory/host_vars/ch-http-proxy.yml
@@ -65,6 +65,7 @@ whawty_auth_store_instances:
       hostname: 192.168.32.1
       port: 3022
       user: sync
+      prometheus: yes
 
 
 whawty_nginx_sso_backends:
diff --git a/inventory/host_vars/ch-pan.yml b/inventory/host_vars/ch-pan.yml
index 16a43695..c6034fa6 100644
--- a/inventory/host_vars/ch-pan.yml
+++ b/inventory/host_vars/ch-pan.yml
@@ -188,6 +188,7 @@ whawty_auth_store_instances:
       hostname: 192.168.32.1
       port: 3022
       user: sync
+      prometheus: yes
 
 whawty_auth_app_instances:
   chaos-at-home:
diff --git a/roles/whawty/auth/store/defaults/main.yml b/roles/whawty/auth/store/defaults/main.yml
index c479c600..5b1ba5a6 100644
--- a/roles/whawty/auth/store/defaults/main.yml
+++ b/roles/whawty/auth/store/defaults/main.yml
@@ -25,3 +25,4 @@
 #       hostname: passwd.example.com
 #       port: 3022
 #       user: sync
+#       prometheus: yes
diff --git a/roles/whawty/auth/store/tasks/sync-client.yml b/roles/whawty/auth/store/tasks/sync-client.yml
index a45e4727..bbd5e8c9 100644
--- a/roles/whawty/auth/store/tasks/sync-client.yml
+++ b/roles/whawty/auth/store/tasks/sync-client.yml
@@ -41,7 +41,15 @@
       {%   endif %}
       {% endif %}
       while true; do
-        /usr/bin/rsync -rtW --delete --delete-delay --delay-updates --partial-dir=.tmp{{ rsync_args | join('') }} -e 'ssh -F "/etc/whawty/auth/.store-{{ item.key }}-sync/ssh_config"' 'rsync://whawty-auth-server/store' '{{ item.value.config.basedir }}'
+        /usr/bin/rsync -rtWi --delete --delete-delay --delay-updates --partial-dir=.tmp{{ rsync_args | join('') }} -e 'ssh -F "/etc/whawty/auth/.store-{{ item.key }}-sync/ssh_config"' 'rsync://whawty-auth-server/store' '{{ item.value.config.basedir }}'
+      {% if (item.value.sync.prometheus | default(False)) %}
+        result=$?
+        now=$(date +"%s")
+        cat <<EOF | sponge /var/lib/prometheus-node-exporter/textfile-collector/whawty-auth-store-sync-{{ item.key }}.prom
+      whawty_auth_store_sync_run{name="{{ item.key }}"} $now
+      whawty_auth_store_sync_exit_code{name="{{ item.key }}"} $result
+      EOF
+      {% endif %}
         sleep 60
       done
     dest: /etc/whawty/auth/.store-{{ item.key }}-sync/run.sh
diff --git a/roles/whawty/auth/store/templates/systemd.service.j2 b/roles/whawty/auth/store/templates/systemd.service.j2
index 7e066901..4a630183 100644
--- a/roles/whawty/auth/store/templates/systemd.service.j2
+++ b/roles/whawty/auth/store/templates/systemd.service.j2
@@ -3,6 +3,7 @@ Description=sync for whawty-auth store {{ item.key }}
 
 [Service]
 Type=simple
+Restart=always
 ExecStart=/etc/whawty/auth/.store-{{ item.key }}-sync/run.sh
 
 # systemd hardening-options
@@ -20,7 +21,7 @@ ProtectHome=true
 ProtectKernelModules=true
 ProtectKernelTunables=true
 ProtectSystem=strict
-ReadWritePaths={{ item.value.config.basedir }}
+ReadWritePaths={{ item.value.config.basedir }}{% if (item.value.sync.prometheus | default(False)) %} /var/lib/prometheus-node-exporter/textfile-collector/{% endif %}{{ '' }}
 RemoveIPC=true
 RestrictNamespaces=true
 RestrictRealtime=true
-- 
cgit v1.2.3