diff options
Diffstat (limited to 'roles/monitoring/prometheus')
7 files changed, 22 insertions, 9 deletions
diff --git a/roles/monitoring/prometheus/alertmanager/defaults/main.yml b/roles/monitoring/prometheus/alertmanager/defaults/main.yml index 47e0ae54..86cd9aa5 100644 --- a/roles/monitoring/prometheus/alertmanager/defaults/main.yml +++ b/roles/monitoring/prometheus/alertmanager/defaults/main.yml @@ -1,6 +1,7 @@ --- prometheus_alertmanager_web_listen_address: 127.0.0.1:9093 # prometheus_alertmanager_web_route_prefix: /alertmanager/ +# prometheus_alertmanager_web_external_url: https://mon.example.com/alertmanager/ prometheus_alertmanager_smtp: smarthost: "127.0.0.1:25" diff --git a/roles/monitoring/prometheus/alertmanager/templates/prometheus-alertmanager.service.j2 b/roles/monitoring/prometheus/alertmanager/templates/prometheus-alertmanager.service.j2 index 5e0e3008..d22d9e01 100644 --- a/roles/monitoring/prometheus/alertmanager/templates/prometheus-alertmanager.service.j2 +++ b/roles/monitoring/prometheus/alertmanager/templates/prometheus-alertmanager.service.j2 @@ -5,7 +5,7 @@ Documentation=https://prometheus.io/docs/alerting/alertmanager/ [Service] Restart=on-failure User=prometheus-alertmanager -ExecStart=/usr/bin/prometheus-alertmanager --config.file=/etc/prometheus/alertmanager.yml --cluster.listen-address= --storage.path="/var/lib/prometheus/alertmanager"{% if prometheus_alertmanager_web_route_prefix is defined %} --web.route-prefix={{ prometheus_alertmanager_web_route_prefix }}{% endif %}{% if prometheus_alertmanager_auth_users is defined %} --web.config.file=/etc/prometheus/alertmanager-web.yml{% endif %} --web.listen-address={{ prometheus_alertmanager_web_listen_address }} +ExecStart=/usr/bin/prometheus-alertmanager --config.file=/etc/prometheus/alertmanager.yml --cluster.listen-address= --storage.path="/var/lib/prometheus/alertmanager"{% if prometheus_alertmanager_web_external_url is defined %} --web.external-url={{ prometheus_alertmanager_web_external_url }}{% endif %}{% if prometheus_alertmanager_web_route_prefix is defined %} --web.route-prefix={{ prometheus_alertmanager_web_route_prefix }}{% endif %}{% if prometheus_alertmanager_auth_users is defined %} --web.config.file=/etc/prometheus/alertmanager-web.yml{% endif %} --web.listen-address={{ prometheus_alertmanager_web_listen_address }} ExecReload=/bin/kill -HUP $MAINPID TimeoutStopSec=20s SendSIGKILL=no diff --git a/roles/monitoring/prometheus/server/defaults/main/main.yml b/roles/monitoring/prometheus/server/defaults/main/main.yml index f74a6f30..99f93e6e 100644 --- a/roles/monitoring/prometheus/server/defaults/main/main.yml +++ b/roles/monitoring/prometheus/server/defaults/main/main.yml @@ -4,6 +4,9 @@ # ... prometheus_server_retention: "15d" +# prometheus_server_external_labels: +# environment: foo +# monitor: {{ inventory_hostname }} prometheus_server_jobs: - node @@ -31,7 +34,8 @@ prometheus_server_rules: # password: geheim prometheus_server_web_listen_address: 127.0.0.1:9090 -# prometheus_server_web_external_url: /prometheus/ +# prometheus_server_web_route_prefix: /prometheus/ +# prometheus_server_web_external_url: https://mon.example.com/prometheus/ # prometheus_server_auth_users: # server: changeme diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_node.yml b/roles/monitoring/prometheus/server/defaults/main/rules_node.yml index 525355d5..d211731a 100644 --- a/roles/monitoring/prometheus/server/defaults/main/rules_node.yml +++ b/roles/monitoring/prometheus/server/defaults/main/rules_node.yml @@ -167,7 +167,7 @@ prometheus_server_rules_node: expr: increase(node_edac_correctable_errors_total[1m]) > 0 for: 0m labels: - severity: info + severity: warning annotations: summary: Host EDAC Correctable Errors detected (instance {{ '{{' }} $labels.instance {{ '}}' }}) description: "Host {{ '{{' }} $labels.instance {{ '}}' }} has had {{ '{{' }} printf \"%.0f\" $value {{ '}}' }} correctable memory errors reported by EDAC in the last 5 minutes.\n VALUE = {{ '{{' }} $value {{ '}}' }}\n LABELS = {{ '{{' }} $labels {{ '}}' }}" @@ -176,7 +176,7 @@ prometheus_server_rules_node: expr: node_edac_uncorrectable_errors_total > 0 for: 0m labels: - severity: warning + severity: critical annotations: summary: Host EDAC Uncorrectable Errors detected (instance {{ '{{' }} $labels.instance {{ '}}' }}) description: "Host {{ '{{' }} $labels.instance {{ '}}' }} has had {{ '{{' }} printf \"%.0f\" $value {{ '}}' }} uncorrectable memory errors reported by EDAC in the last 5 minutes.\n VALUE = {{ '{{' }} $value {{ '}}' }}\n LABELS = {{ '{{' }} $labels {{ '}}' }}" @@ -206,7 +206,7 @@ prometheus_server_rules_node: severity: warning annotations: summary: Host Network Interface Saturated (instance {{ '{{' }} $labels.instance {{ '}}' }}) - description: "The network interface \"{{ '{{' }} $labels.interface {{ '}}' }}\" on \"{{ '{{' }} $labels.instance {{ '}}' }}\" is getting overloaded.\n VALUE = {{ '{{' }} $value {{ '}}' }}\n LABELS = {{ '{{' }} $labels {{ '}}' }}" + description: "The network interface \"{{ '{{' }} $labels.device {{ '}}' }}\" on \"{{ '{{' }} $labels.instance {{ '}}' }}\" is getting overloaded.\n VALUE = {{ '{{' }} $value {{ '}}' }}\n LABELS = {{ '{{' }} $labels {{ '}}' }}" - alert: HostConntrackLimit expr: node_nf_conntrack_entries / node_nf_conntrack_entries_limit > 0.8 diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_prometheus.yml b/roles/monitoring/prometheus/server/defaults/main/rules_prometheus.yml index 8d4672b1..422f84cb 100644 --- a/roles/monitoring/prometheus/server/defaults/main/rules_prometheus.yml +++ b/roles/monitoring/prometheus/server/defaults/main/rules_prometheus.yml @@ -215,7 +215,9 @@ prometheus_server_rules_prometheus_alertmanager: expr: vector(1) for: 0m labels: - severity: critical + severity: informational + instance: prometheus + timeout: 7200 annotations: summary: Prometheus AlertManager E2E dead man switch (instance {{ '{{' }} $labels.instance {{ '}}' }}) description: "Prometheus DeadManSwitch is an always-firing alert. It's used as an end-to-end test of Prometheus through the Alertmanager.\n VALUE = {{ '{{' }} $value {{ '}}' }}\n LABELS = {{ '{{' }} $labels {{ '}}' }}" diff --git a/roles/monitoring/prometheus/server/templates/prometheus.service.j2 b/roles/monitoring/prometheus/server/templates/prometheus.service.j2 index 77a3b02a..e65e9425 100644 --- a/roles/monitoring/prometheus/server/templates/prometheus.service.j2 +++ b/roles/monitoring/prometheus/server/templates/prometheus.service.j2 @@ -6,7 +6,7 @@ After=time-sync.target [Service] Restart=on-failure User=prometheus -ExecStart=/usr/bin/prometheus --config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/var/lib/prometheus/metrics2/ --storage.tsdb.retention.time={{ prometheus_server_retention }}{% if prometheus_server_web_external_url is defined %} --web.external-url={{ prometheus_server_web_external_url }}{% endif %}{% if prometheus_server_auth_users is defined %} --web.config.file=/etc/prometheus/prometheus-web.yml{% endif %} --web.listen-address={{ prometheus_server_web_listen_address }} +ExecStart=/usr/bin/prometheus --config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/var/lib/prometheus/metrics2/ --storage.tsdb.retention.time={{ prometheus_server_retention }}{% if prometheus_server_web_external_url is defined %} --web.external-url={{ prometheus_server_web_external_url }}{% endif %}{% if prometheus_server_web_route_prefix is defined %} --web.route-prefix={{ prometheus_server_web_route_prefix }}{% endif %}{% if prometheus_server_auth_users is defined %} --web.config.file=/etc/prometheus/prometheus-web.yml{% endif %} --web.listen-address={{ prometheus_server_web_listen_address }} ExecReload=/bin/kill -HUP $MAINPID TimeoutStopSec=20s SendSIGKILL=no diff --git a/roles/monitoring/prometheus/server/templates/prometheus.yml.j2 b/roles/monitoring/prometheus/server/templates/prometheus.yml.j2 index 883aa223..aed69de5 100644 --- a/roles/monitoring/prometheus/server/templates/prometheus.yml.j2 +++ b/roles/monitoring/prometheus/server/templates/prometheus.yml.j2 @@ -3,6 +3,12 @@ global: scrape_interval: 15s evaluation_interval: 15s +{% if prometheus_server_external_labels is defined %} + external_labels: +{% for name, value in prometheus_server_external_labels.items() %} + {{ name }}: {{ value }} +{% endfor %} +{% endif %} rule_files: - /etc/prometheus/rules/*.yml @@ -27,8 +33,8 @@ alerting: scrape_configs: - job_name: 'prometheus' -{% if prometheus_server_web_external_url is defined %} - metrics_path: '{{ (prometheus_server_web_external_url | urlsplit('path'), 'metrics') | path_join }}' +{% if prometheus_server_web_route_prefix is defined or prometheus_server_web_external_url is defined %} + metrics_path: '{{ (prometheus_server_web_route_prefix | default(prometheus_server_web_external_url | default('') | urlsplit('path')), 'metrics') | path_join }}' {% endif %} {% if prometheus_server_selfscraping_auth is defined %} basic_auth: |