diff options
8 files changed, 27 insertions, 10 deletions
diff --git a/inventory/host_vars/ch-mon.yml b/inventory/host_vars/ch-mon.yml index 0c27a522..c16398bc 100644 --- a/inventory/host_vars/ch-mon.yml +++ b/inventory/host_vars/ch-mon.yml @@ -61,6 +61,10 @@ prometheus_server_storage: size: 30G fs: ext4 +prometheus_server_external_labels: + environment: chaos-at-home + monitor: "{{ inventory_hostname }}" + prometheus_server_alertmanager: url: "127.0.0.1:9093" path_prefix: "/alertmanager/" @@ -68,7 +72,7 @@ prometheus_server_alertmanager: username: server password: "{{ vault_prometheus_alertmanager_auth_user_passwords['server'] }}" -prometheus_server_web_external_url: /prometheus/ +prometheus_server_web_external_url: "http://{{ network.primary.address | ipaddr('address') }}/prometheus/" prometheus_server_auth_users: server: "{{ vault_prometheus_server_auth_user_passwords['server'] }}" @@ -136,7 +140,7 @@ prometheus_alertmanager_smtp: from: "noreply@chaos-at-home.org" require_tls: no -prometheus_alertmanager_web_route_prefix: /alertmanager/ +prometheus_alertmanager_web_external_url: "http://{{ network.primary.address | ipaddr('address') }}/alertmanager/" prometheus_alertmanager_auth_users: server: "{{ vault_prometheus_alertmanager_auth_user_passwords['server'] }}" diff --git a/inventory/host_vars/ele-mon.yml b/inventory/host_vars/ele-mon.yml index 4ad37bce..5d03837c 100644 --- a/inventory/host_vars/ele-mon.yml +++ b/inventory/host_vars/ele-mon.yml @@ -70,7 +70,7 @@ prometheus_server_alertmanager: url: "127.0.0.1:9093" path_prefix: "/alertmanager/" -prometheus_server_web_external_url: /prometheus/ +prometheus_server_web_external_url: "http://{{ network.primary.address | ipaddr('address') }}/prometheus/" prometheus_exporters_extra: @@ -101,7 +101,7 @@ prometheus_exporter_mikrotik_devices: user: prometheus password: "{{ vault_prometheus_exporter_mikrotik_api_password }}" -prometheus_alertmanager_web_route_prefix: /alertmanager/ +prometheus_alertmanager_web_external_url: "http://{{ network.primary.address | ipaddr('address') }}/alertmanager/" prometheus_alertmanager_smtp: smarthost: "mailrelay.chaos-at-home.org:587" from: "noreply@elev8.at" diff --git a/roles/monitoring/prometheus/alertmanager/defaults/main.yml b/roles/monitoring/prometheus/alertmanager/defaults/main.yml index 47e0ae54..86cd9aa5 100644 --- a/roles/monitoring/prometheus/alertmanager/defaults/main.yml +++ b/roles/monitoring/prometheus/alertmanager/defaults/main.yml @@ -1,6 +1,7 @@ --- prometheus_alertmanager_web_listen_address: 127.0.0.1:9093 # prometheus_alertmanager_web_route_prefix: /alertmanager/ +# prometheus_alertmanager_web_external_url: https://mon.example.com/alertmanager/ prometheus_alertmanager_smtp: smarthost: "127.0.0.1:25" diff --git a/roles/monitoring/prometheus/alertmanager/templates/prometheus-alertmanager.service.j2 b/roles/monitoring/prometheus/alertmanager/templates/prometheus-alertmanager.service.j2 index 5e0e3008..d22d9e01 100644 --- a/roles/monitoring/prometheus/alertmanager/templates/prometheus-alertmanager.service.j2 +++ b/roles/monitoring/prometheus/alertmanager/templates/prometheus-alertmanager.service.j2 @@ -5,7 +5,7 @@ Documentation=https://prometheus.io/docs/alerting/alertmanager/ [Service] Restart=on-failure User=prometheus-alertmanager -ExecStart=/usr/bin/prometheus-alertmanager --config.file=/etc/prometheus/alertmanager.yml --cluster.listen-address= --storage.path="/var/lib/prometheus/alertmanager"{% if prometheus_alertmanager_web_route_prefix is defined %} --web.route-prefix={{ prometheus_alertmanager_web_route_prefix }}{% endif %}{% if prometheus_alertmanager_auth_users is defined %} --web.config.file=/etc/prometheus/alertmanager-web.yml{% endif %} --web.listen-address={{ prometheus_alertmanager_web_listen_address }} +ExecStart=/usr/bin/prometheus-alertmanager --config.file=/etc/prometheus/alertmanager.yml --cluster.listen-address= --storage.path="/var/lib/prometheus/alertmanager"{% if prometheus_alertmanager_web_external_url is defined %} --web.external-url={{ prometheus_alertmanager_web_external_url }}{% endif %}{% if prometheus_alertmanager_web_route_prefix is defined %} --web.route-prefix={{ prometheus_alertmanager_web_route_prefix }}{% endif %}{% if prometheus_alertmanager_auth_users is defined %} --web.config.file=/etc/prometheus/alertmanager-web.yml{% endif %} --web.listen-address={{ prometheus_alertmanager_web_listen_address }} ExecReload=/bin/kill -HUP $MAINPID TimeoutStopSec=20s SendSIGKILL=no diff --git a/roles/monitoring/prometheus/server/defaults/main/main.yml b/roles/monitoring/prometheus/server/defaults/main/main.yml index f74a6f30..99f93e6e 100644 --- a/roles/monitoring/prometheus/server/defaults/main/main.yml +++ b/roles/monitoring/prometheus/server/defaults/main/main.yml @@ -4,6 +4,9 @@ # ... prometheus_server_retention: "15d" +# prometheus_server_external_labels: +# environment: foo +# monitor: {{ inventory_hostname }} prometheus_server_jobs: - node @@ -31,7 +34,8 @@ prometheus_server_rules: # password: geheim prometheus_server_web_listen_address: 127.0.0.1:9090 -# prometheus_server_web_external_url: /prometheus/ +# prometheus_server_web_route_prefix: /prometheus/ +# prometheus_server_web_external_url: https://mon.example.com/prometheus/ # prometheus_server_auth_users: # server: changeme diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_prometheus.yml b/roles/monitoring/prometheus/server/defaults/main/rules_prometheus.yml index 8d4672b1..422f84cb 100644 --- a/roles/monitoring/prometheus/server/defaults/main/rules_prometheus.yml +++ b/roles/monitoring/prometheus/server/defaults/main/rules_prometheus.yml @@ -215,7 +215,9 @@ prometheus_server_rules_prometheus_alertmanager: expr: vector(1) for: 0m labels: - severity: critical + severity: informational + instance: prometheus + timeout: 7200 annotations: summary: Prometheus AlertManager E2E dead man switch (instance {{ '{{' }} $labels.instance {{ '}}' }}) description: "Prometheus DeadManSwitch is an always-firing alert. It's used as an end-to-end test of Prometheus through the Alertmanager.\n VALUE = {{ '{{' }} $value {{ '}}' }}\n LABELS = {{ '{{' }} $labels {{ '}}' }}" diff --git a/roles/monitoring/prometheus/server/templates/prometheus.service.j2 b/roles/monitoring/prometheus/server/templates/prometheus.service.j2 index 77a3b02a..e65e9425 100644 --- a/roles/monitoring/prometheus/server/templates/prometheus.service.j2 +++ b/roles/monitoring/prometheus/server/templates/prometheus.service.j2 @@ -6,7 +6,7 @@ After=time-sync.target [Service] Restart=on-failure User=prometheus -ExecStart=/usr/bin/prometheus --config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/var/lib/prometheus/metrics2/ --storage.tsdb.retention.time={{ prometheus_server_retention }}{% if prometheus_server_web_external_url is defined %} --web.external-url={{ prometheus_server_web_external_url }}{% endif %}{% if prometheus_server_auth_users is defined %} --web.config.file=/etc/prometheus/prometheus-web.yml{% endif %} --web.listen-address={{ prometheus_server_web_listen_address }} +ExecStart=/usr/bin/prometheus --config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/var/lib/prometheus/metrics2/ --storage.tsdb.retention.time={{ prometheus_server_retention }}{% if prometheus_server_web_external_url is defined %} --web.external-url={{ prometheus_server_web_external_url }}{% endif %}{% if prometheus_server_web_route_prefix is defined %} --web.route-prefix={{ prometheus_server_web_route_prefix }}{% endif %}{% if prometheus_server_auth_users is defined %} --web.config.file=/etc/prometheus/prometheus-web.yml{% endif %} --web.listen-address={{ prometheus_server_web_listen_address }} ExecReload=/bin/kill -HUP $MAINPID TimeoutStopSec=20s SendSIGKILL=no diff --git a/roles/monitoring/prometheus/server/templates/prometheus.yml.j2 b/roles/monitoring/prometheus/server/templates/prometheus.yml.j2 index 883aa223..aed69de5 100644 --- a/roles/monitoring/prometheus/server/templates/prometheus.yml.j2 +++ b/roles/monitoring/prometheus/server/templates/prometheus.yml.j2 @@ -3,6 +3,12 @@ global: scrape_interval: 15s evaluation_interval: 15s +{% if prometheus_server_external_labels is defined %} + external_labels: +{% for name, value in prometheus_server_external_labels.items() %} + {{ name }}: {{ value }} +{% endfor %} +{% endif %} rule_files: - /etc/prometheus/rules/*.yml @@ -27,8 +33,8 @@ alerting: scrape_configs: - job_name: 'prometheus' -{% if prometheus_server_web_external_url is defined %} - metrics_path: '{{ (prometheus_server_web_external_url | urlsplit('path'), 'metrics') | path_join }}' +{% if prometheus_server_web_route_prefix is defined or prometheus_server_web_external_url is defined %} + metrics_path: '{{ (prometheus_server_web_route_prefix | default(prometheus_server_web_external_url | default('') | urlsplit('path')), 'metrics') | path_join }}' {% endif %} {% if prometheus_server_selfscraping_auth is defined %} basic_auth: |