From 1bbe8582d9650eda816f9c596d8cd3b3fe9998e7 Mon Sep 17 00:00:00 2001
From: Christian Pointner <equinox@spreadspace.org>
Date: Sun, 6 Mar 2022 00:31:22 +0100
Subject: alerta: test alertmanager silence integration

---
 inventory/host_vars/ch-mon.yml                                  | 3 ++-
 roles/monitoring/alerta/tasks/main.yml                          | 6 +++++-
 roles/monitoring/prometheus/server/defaults/main/rules_node.yml | 2 +-
 3 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/inventory/host_vars/ch-mon.yml b/inventory/host_vars/ch-mon.yml
index bd952fc8..684cc722 100644
--- a/inventory/host_vars/ch-mon.yml
+++ b/inventory/host_vars/ch-mon.yml
@@ -145,6 +145,7 @@ prometheus_alertmanager_web_external_url: "http://{{ network.primary.address | i
 prometheus_alertmanager_auth_users:
   server: "{{ vault_prometheus_alertmanager_auth_user_passwords['server'] }}"
   admin: "{{ vault_prometheus_alertmanager_auth_user_passwords['admin'] }}"
+  alerta: "alerta" ## TODO: move this to vault
 
 prometheus_alertmanager_route:
   receiver: alerta
@@ -161,7 +162,7 @@ prometheus_alertmanager_receivers:
       http_config:
         basic_auth:
           username: admin
-          password: alerta
+          password: alerta ## TODO: move this to vault
 
 
 grafana_secret_key: "{{ vault_grafana_secret_key }}"
diff --git a/roles/monitoring/alerta/tasks/main.yml b/roles/monitoring/alerta/tasks/main.yml
index af7530d8..a4fd3df9 100644
--- a/roles/monitoring/alerta/tasks/main.yml
+++ b/roles/monitoring/alerta/tasks/main.yml
@@ -12,10 +12,14 @@
     content: |
       DEBUG = {{ '{{' }} 'True' if env.DEBUG else 'False' {{ '}}' }}
       SECRET = "{{ '{{' }} env.SECRET_KEY {{ '}}' }}"
-      PLUGINS = ['reject', 'blackout', 'heartbeat']
+      PLUGINS = ['reject', 'blackout', 'heartbeat', 'prometheus']
       DEFAULT_ENVIRONMENT = 'unknown'
       ALLOWED_ENVIRONMENTS = ['unknown', 'chaos-at-.*']
       HEARTBEAT_EVENTS = ['PrometheusAlertmanagerE2eDeadManSwitch']
+      ALERTMANAGER_USERNAME = 'alerta'
+      ALERTMANAGER_PASSWORD = 'alerta'
+      ALERTMANAGER_SILENCE_FROM_ACK = True
+      ALERTMANAGER_USE_EXTERNALURL_FOR_SILENCES = True
     dest: "{{ alerta_base_path }}/config/alertad.conf.j2"
 
 - name: install pod manifest
diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_node.yml b/roles/monitoring/prometheus/server/defaults/main/rules_node.yml
index 525355d5..75e96bca 100644
--- a/roles/monitoring/prometheus/server/defaults/main/rules_node.yml
+++ b/roles/monitoring/prometheus/server/defaults/main/rules_node.yml
@@ -206,7 +206,7 @@ prometheus_server_rules_node:
       severity: warning
     annotations:
       summary: Host Network Interface Saturated (instance {{ '{{' }} $labels.instance {{ '}}' }})
-      description: "The network interface \"{{ '{{' }} $labels.interface {{ '}}' }}\" on \"{{ '{{' }} $labels.instance {{ '}}' }}\" is getting overloaded.\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+      description: "The network interface \"{{ '{{' }} $labels.device {{ '}}' }}\" on \"{{ '{{' }} $labels.instance {{ '}}' }}\" is getting overloaded.\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
 
   - alert: HostConntrackLimit
     expr: node_nf_conntrack_entries / node_nf_conntrack_entries_limit > 0.8
-- 
cgit v1.2.3