diff options
| -rw-r--r-- | inventory/group_vars/promzone-chaos-at-home/vars.yml | 20 | ||||
| -rw-r--r-- | inventory/host_vars/ch-http-proxy.yml | 1 | ||||
| -rw-r--r-- | inventory/host_vars/ch-pan.yml | 1 | ||||
| -rw-r--r-- | roles/whawty/auth/store/defaults/main.yml | 1 | ||||
| -rw-r--r-- | roles/whawty/auth/store/tasks/sync-client.yml | 10 | ||||
| -rw-r--r-- | roles/whawty/auth/store/templates/systemd.service.j2 | 3 |
6 files changed, 34 insertions, 2 deletions
diff --git a/inventory/group_vars/promzone-chaos-at-home/vars.yml b/inventory/group_vars/promzone-chaos-at-home/vars.yml index 8c3b7945..1e3c2ada 100644 --- a/inventory/group_vars/promzone-chaos-at-home/vars.yml +++ b/inventory/group_vars/promzone-chaos-at-home/vars.yml @@ -41,3 +41,23 @@ prometheus_zone_name: chaos@home ## TODO: ## potential extra alert rule: (bind_zone_serial{instance="ch-mimas"} == bool on(job, view, zone_name) bind_zone_serial{instance="ch-pan"}) != 1 + + +prometheus_server_rules_whawty_nginx_sso_extra: + - alert: WhawtyAuthStoreSyncTooLongAgo + expr: time() - whawty_auth_store_sync_run > 3600 + for: 0m + labels: + severity: warning + annotations: + summary: The last whawty-auth store sync was too long ago (instance {{ '{{' }} $labels.instance {{ '}}' }}) + description: "The last whawty-auth store sync of {{ '{{' }} $labels.name {{ '}}' }} on {{ '{{' }} $labels.instance {{ '}}' }} ran more then an hours ago.\n VALUE = {{ '{{' }} $value {{ '}}' }}\n LABELS = {{ '{{' }} $labels {{ '}}' }}" + + - alert: WhawtyAuthStoreSyncFailed + expr: whawty_auth_store_sync_exit_code != 0 + for: 0m + labels: + severity: warning + annotations: + summary: The last whawty-auth sync failed (instance {{ '{{' }} $labels.instance {{ '}}' }}) + description: "The last whawty-auth store sync of {{ '{{' }} $labels.name {{ '}}' }} on {{ '{{' }} $labels.instance {{ '}}' }} has failed.\n VALUE = {{ '{{' }} $value {{ '}}' }}\n LABELS = {{ '{{' }} $labels {{ '}}' }}" diff --git a/inventory/host_vars/ch-http-proxy.yml b/inventory/host_vars/ch-http-proxy.yml index 46e63c1d..5be067ec 100644 --- a/inventory/host_vars/ch-http-proxy.yml +++ b/inventory/host_vars/ch-http-proxy.yml @@ -65,6 +65,7 @@ whawty_auth_store_instances: hostname: 192.168.32.1 port: 3022 user: sync + prometheus: yes whawty_nginx_sso_backends: diff --git a/inventory/host_vars/ch-pan.yml b/inventory/host_vars/ch-pan.yml index 16a43695..c6034fa6 100644 --- a/inventory/host_vars/ch-pan.yml +++ b/inventory/host_vars/ch-pan.yml @@ -188,6 +188,7 @@ whawty_auth_store_instances: hostname: 192.168.32.1 port: 3022 user: sync + prometheus: yes whawty_auth_app_instances: chaos-at-home: diff --git a/roles/whawty/auth/store/defaults/main.yml b/roles/whawty/auth/store/defaults/main.yml index c479c600..5b1ba5a6 100644 --- a/roles/whawty/auth/store/defaults/main.yml +++ b/roles/whawty/auth/store/defaults/main.yml @@ -25,3 +25,4 @@ # hostname: passwd.example.com # port: 3022 # user: sync +# prometheus: yes diff --git a/roles/whawty/auth/store/tasks/sync-client.yml b/roles/whawty/auth/store/tasks/sync-client.yml index a45e4727..bbd5e8c9 100644 --- a/roles/whawty/auth/store/tasks/sync-client.yml +++ b/roles/whawty/auth/store/tasks/sync-client.yml @@ -41,7 +41,15 @@ {% endif %} {% endif %} while true; do - /usr/bin/rsync -rtW --delete --delete-delay --delay-updates --partial-dir=.tmp{{ rsync_args | join('') }} -e 'ssh -F "/etc/whawty/auth/.store-{{ item.key }}-sync/ssh_config"' 'rsync://whawty-auth-server/store' '{{ item.value.config.basedir }}' + /usr/bin/rsync -rtWi --delete --delete-delay --delay-updates --partial-dir=.tmp{{ rsync_args | join('') }} -e 'ssh -F "/etc/whawty/auth/.store-{{ item.key }}-sync/ssh_config"' 'rsync://whawty-auth-server/store' '{{ item.value.config.basedir }}' + {% if (item.value.sync.prometheus | default(False)) %} + result=$? + now=$(date +"%s") + cat <<EOF | sponge /var/lib/prometheus-node-exporter/textfile-collector/whawty-auth-store-sync-{{ item.key }}.prom + whawty_auth_store_sync_run{name="{{ item.key }}"} $now + whawty_auth_store_sync_exit_code{name="{{ item.key }}"} $result + EOF + {% endif %} sleep 60 done dest: /etc/whawty/auth/.store-{{ item.key }}-sync/run.sh diff --git a/roles/whawty/auth/store/templates/systemd.service.j2 b/roles/whawty/auth/store/templates/systemd.service.j2 index 7e066901..4a630183 100644 --- a/roles/whawty/auth/store/templates/systemd.service.j2 +++ b/roles/whawty/auth/store/templates/systemd.service.j2 @@ -3,6 +3,7 @@ Description=sync for whawty-auth store {{ item.key }} [Service] Type=simple +Restart=always ExecStart=/etc/whawty/auth/.store-{{ item.key }}-sync/run.sh # systemd hardening-options @@ -20,7 +21,7 @@ ProtectHome=true ProtectKernelModules=true ProtectKernelTunables=true ProtectSystem=strict -ReadWritePaths={{ item.value.config.basedir }} +ReadWritePaths={{ item.value.config.basedir }}{% if (item.value.sync.prometheus | default(False)) %} /var/lib/prometheus-node-exporter/textfile-collector/{% endif %}{{ '' }} RemoveIPC=true RestrictNamespaces=true RestrictRealtime=true |