prometheus: cleanups and fixes in prep for alerta

6 files changed, 19 insertions, 6 deletions

diff --git a/roles/monitoring/prometheus/alertmanager/defaults/main.yml b/roles/monitoring/prometheus/alertmanager/defaults/main.yml
index 47e0ae54..86cd9aa5 100644
--- a/roles/monitoring/prometheus/alertmanager/defaults/main.yml
+++ b/roles/monitoring/prometheus/alertmanager/defaults/main.yml
@@ -1,6 +1,7 @@
 ---
 prometheus_alertmanager_web_listen_address: 127.0.0.1:9093
 # prometheus_alertmanager_web_route_prefix: /alertmanager/
+# prometheus_alertmanager_web_external_url: https://mon.example.com/alertmanager/
 
 prometheus_alertmanager_smtp:
   smarthost: "127.0.0.1:25"
diff --git a/roles/monitoring/prometheus/alertmanager/templates/prometheus-alertmanager.service.j2 b/roles/monitoring/prometheus/alertmanager/templates/prometheus-alertmanager.service.j2
index 5e0e3008..d22d9e01 100644
--- a/roles/monitoring/prometheus/alertmanager/templates/prometheus-alertmanager.service.j2
+++ b/roles/monitoring/prometheus/alertmanager/templates/prometheus-alertmanager.service.j2
@@ -5,7 +5,7 @@ Documentation=https://prometheus.io/docs/alerting/alertmanager/
 [Service]
 Restart=on-failure
 User=prometheus-alertmanager
-ExecStart=/usr/bin/prometheus-alertmanager --config.file=/etc/prometheus/alertmanager.yml --cluster.listen-address= --storage.path="/var/lib/prometheus/alertmanager"{% if prometheus_alertmanager_web_route_prefix is defined %} --web.route-prefix={{ prometheus_alertmanager_web_route_prefix }}{% endif %}{% if prometheus_alertmanager_auth_users is defined %} --web.config.file=/etc/prometheus/alertmanager-web.yml{% endif %} --web.listen-address={{ prometheus_alertmanager_web_listen_address }}
+ExecStart=/usr/bin/prometheus-alertmanager --config.file=/etc/prometheus/alertmanager.yml --cluster.listen-address= --storage.path="/var/lib/prometheus/alertmanager"{% if prometheus_alertmanager_web_external_url is defined %} --web.external-url={{ prometheus_alertmanager_web_external_url }}{% endif %}{% if prometheus_alertmanager_web_route_prefix is defined %} --web.route-prefix={{ prometheus_alertmanager_web_route_prefix }}{% endif %}{% if prometheus_alertmanager_auth_users is defined %} --web.config.file=/etc/prometheus/alertmanager-web.yml{% endif %} --web.listen-address={{ prometheus_alertmanager_web_listen_address }}
 ExecReload=/bin/kill -HUP $MAINPID
 TimeoutStopSec=20s
 SendSIGKILL=no
diff --git a/roles/monitoring/prometheus/server/defaults/main/main.yml b/roles/monitoring/prometheus/server/defaults/main/main.yml
index f74a6f30..99f93e6e 100644
--- a/roles/monitoring/prometheus/server/defaults/main/main.yml
+++ b/roles/monitoring/prometheus/server/defaults/main/main.yml
@@ -4,6 +4,9 @@
 #   ...
 
 prometheus_server_retention: "15d"
+# prometheus_server_external_labels:
+#   environment: foo
+#   monitor: {{ inventory_hostname }}
 
 prometheus_server_jobs:
   - node
@@ -31,7 +34,8 @@ prometheus_server_rules:
 #     password: geheim
 
 prometheus_server_web_listen_address: 127.0.0.1:9090
-# prometheus_server_web_external_url: /prometheus/
+# prometheus_server_web_route_prefix: /prometheus/
+# prometheus_server_web_external_url: https://mon.example.com/prometheus/
 
 # prometheus_server_auth_users:
 #   server: changeme
diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_prometheus.yml b/roles/monitoring/prometheus/server/defaults/main/rules_prometheus.yml
index 8d4672b1..422f84cb 100644
--- a/roles/monitoring/prometheus/server/defaults/main/rules_prometheus.yml
+++ b/roles/monitoring/prometheus/server/defaults/main/rules_prometheus.yml
@@ -215,7 +215,9 @@ prometheus_server_rules_prometheus_alertmanager:
     expr: vector(1)
     for: 0m
     labels:
-      severity: critical
+      severity: informational
+      instance: prometheus
+      timeout: 7200
     annotations:
       summary: Prometheus AlertManager E2E dead man switch (instance {{ '{{' }} $labels.instance {{ '}}' }})
       description: "Prometheus DeadManSwitch is an always-firing alert. It's used as an end-to-end test of Prometheus through the Alertmanager.\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
diff --git a/roles/monitoring/prometheus/server/templates/prometheus.service.j2 b/roles/monitoring/prometheus/server/templates/prometheus.service.j2
index 77a3b02a..e65e9425 100644
--- a/roles/monitoring/prometheus/server/templates/prometheus.service.j2
+++ b/roles/monitoring/prometheus/server/templates/prometheus.service.j2
@@ -6,7 +6,7 @@ After=time-sync.target
 [Service]
 Restart=on-failure
 User=prometheus
-ExecStart=/usr/bin/prometheus --config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/var/lib/prometheus/metrics2/ --storage.tsdb.retention.time={{ prometheus_server_retention }}{% if prometheus_server_web_external_url is defined %} --web.external-url={{ prometheus_server_web_external_url }}{% endif %}{% if prometheus_server_auth_users is defined %} --web.config.file=/etc/prometheus/prometheus-web.yml{% endif %} --web.listen-address={{ prometheus_server_web_listen_address }}
+ExecStart=/usr/bin/prometheus --config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/var/lib/prometheus/metrics2/ --storage.tsdb.retention.time={{ prometheus_server_retention }}{% if prometheus_server_web_external_url is defined %} --web.external-url={{ prometheus_server_web_external_url }}{% endif %}{% if prometheus_server_web_route_prefix is defined %} --web.route-prefix={{ prometheus_server_web_route_prefix }}{% endif %}{% if prometheus_server_auth_users is defined %} --web.config.file=/etc/prometheus/prometheus-web.yml{% endif %} --web.listen-address={{ prometheus_server_web_listen_address }}
 ExecReload=/bin/kill -HUP $MAINPID
 TimeoutStopSec=20s
 SendSIGKILL=no
diff --git a/roles/monitoring/prometheus/server/templates/prometheus.yml.j2 b/roles/monitoring/prometheus/server/templates/prometheus.yml.j2
index 883aa223..aed69de5 100644
--- a/roles/monitoring/prometheus/server/templates/prometheus.yml.j2
+++ b/roles/monitoring/prometheus/server/templates/prometheus.yml.j2
@@ -3,6 +3,12 @@
 global:
   scrape_interval:     15s
   evaluation_interval: 15s
+{% if prometheus_server_external_labels is defined %}
+  external_labels:
+{%   for name, value in prometheus_server_external_labels.items() %}
+    {{ name }}: {{ value }}
+{%   endfor %}
+{% endif %}
 
 rule_files:
   - /etc/prometheus/rules/*.yml
@@ -27,8 +33,8 @@ alerting:
 
 scrape_configs:
   - job_name: 'prometheus'
-{% if prometheus_server_web_external_url is defined %}
-    metrics_path: '{{ (prometheus_server_web_external_url | urlsplit('path'), 'metrics') | path_join }}'
+{% if prometheus_server_web_route_prefix is defined or prometheus_server_web_external_url is defined %}
+    metrics_path: '{{ (prometheus_server_web_route_prefix | default(prometheus_server_web_external_url | default('') | urlsplit('path')), 'metrics') | path_join }}'
 {% endif %}
 {% if prometheus_server_selfscraping_auth is defined %}
     basic_auth: