summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorChristian Pointner <equinox@spreadspace.org>2022-03-16 21:39:40 +0100
committerChristian Pointner <equinox@spreadspace.org>2022-03-16 21:39:40 +0100
commit8abc6406b3365fc8d6f25e293aee2b56c4ca611a (patch)
tree7d9f4643fa16c3b2e7960894fe78159bdc7158b3
parentzfs/syncoid-autosuspend: also check for resilvering (diff)
alerta: tweak blackout settings
-rw-r--r--roles/monitoring/alerta/defaults/main.yml8
-rw-r--r--roles/monitoring/alerta/templates/alertad.conf.j2.j24
-rw-r--r--roles/monitoring/prometheus/server/defaults/main/rules_prometheus.yml2
3 files changed, 11 insertions, 3 deletions
diff --git a/roles/monitoring/alerta/defaults/main.yml b/roles/monitoring/alerta/defaults/main.yml
index af631cb2..315c2c3f 100644
--- a/roles/monitoring/alerta/defaults/main.yml
+++ b/roles/monitoring/alerta/defaults/main.yml
@@ -14,8 +14,8 @@ alerta_web_port: 8080
# alerta_postgres_password: "secret"
-alerta_alert_timeout: 86400
-alerta_heartbeat_timeout: 7200
+alerta_alert_timeout: 3600
+alerta_heartbeat_timeout: 1200
alerta_default_environment: Development
alerta_environments:
@@ -27,6 +27,10 @@ alerta_plugins:
- reject
- blackout
+alerta_plugin_blackout:
+ set_status: True
+ accept: ['normal', 'ok', 'cleared']
+
# alerta_plugin_heartbeat:
# events:
# - PrometheusAlertmanagerE2eDeadManSwitch
diff --git a/roles/monitoring/alerta/templates/alertad.conf.j2.j2 b/roles/monitoring/alerta/templates/alertad.conf.j2.j2
index b013f480..cf0b2426 100644
--- a/roles/monitoring/alerta/templates/alertad.conf.j2.j2
+++ b/roles/monitoring/alerta/templates/alertad.conf.j2.j2
@@ -5,6 +5,10 @@ HEARTBEAT_TIMEOUT = {{ alerta_heartbeat_timeout }}
DEFAULT_ENVIRONMENT = '{{ alerta_default_environment }}'
ALLOWED_ENVIRONMENTS = ['{{ alerta_environments | join("', '") }}']
PLUGINS = ['{{ alerta_plugins | join("', '") }}']
+{% if 'blackout' in alerta_plugins %}
+NOTIFICATION_BLACKOUT = {{ 'True' if alerta_plugin_blackout.set_status else 'False' }}
+BLACKOUT_ACCEPT = ['{{ alerta_plugin_blackout.accept | join("', '") }}']
+{% endif %}
{% if 'heartbeat' in alerta_plugins %}
HEARTBEAT_EVENTS = ['{{ alerta_plugin_heartbeat.events | join("', '") }}']
{% endif %}
diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_prometheus.yml b/roles/monitoring/prometheus/server/defaults/main/rules_prometheus.yml
index 422f84cb..b568f722 100644
--- a/roles/monitoring/prometheus/server/defaults/main/rules_prometheus.yml
+++ b/roles/monitoring/prometheus/server/defaults/main/rules_prometheus.yml
@@ -217,7 +217,7 @@ prometheus_server_rules_prometheus_alertmanager:
labels:
severity: informational
instance: prometheus
- timeout: 7200
+ timeout: 1200
annotations:
summary: Prometheus AlertManager E2E dead man switch (instance {{ '{{' }} $labels.instance {{ '}}' }})
description: "Prometheus DeadManSwitch is an always-firing alert. It's used as an end-to-end test of Prometheus through the Alertmanager.\n VALUE = {{ '{{' }} $value {{ '}}' }}\n LABELS = {{ '{{' }} $labels {{ '}}' }}"