diff options
Diffstat (limited to 'roles/monitoring/prometheus/server')
4 files changed, 65 insertions, 3 deletions
diff --git a/roles/monitoring/prometheus/server/defaults/main/main.yml b/roles/monitoring/prometheus/server/defaults/main/main.yml index 9f867568..dd290e9e 100644 --- a/roles/monitoring/prometheus/server/defaults/main/main.yml +++ b/roles/monitoring/prometheus/server/defaults/main/main.yml @@ -15,7 +15,7 @@ prometheus_server_evaluation_interval: "15s" prometheus_server_jobs: - node -#prometheus_server_jobs_extra: | +#prometheus_server_jobs_extra: |- # - job_name: ... prometheus_server_rules: @@ -41,6 +41,7 @@ prometheus_server_rules: nftables: "{{ prometheus_server_rules_nftables + prometheus_server_rules_nftables_extra }}" whawty-nginx-sso: "{{ prometheus_server_rules_whawty_nginx_sso + prometheus_server_rules_whawty_nginx_sso_extra }}" mosquitto: "{{ prometheus_server_rules_mosquitto + prometheus_server_rules_mosquitto_extra }}" + coredns: "{{ prometheus_server_rules_coredns + prometheus_server_rules_coredns_extra }}" # prometheus_server_alertmanager: # url: "127.0.0.1:9093" @@ -48,6 +49,7 @@ prometheus_server_rules: # basic_auth: # username: server # password: geheim +# scrape_instance: <inventory-hostname> prometheus_server_web_listen_address: 127.0.0.1:9090 # prometheus_server_web_route_prefix: /prometheus/ @@ -73,5 +75,27 @@ prometheus_server_web_listen_address: 127.0.0.1:9090 # - node # - blackbox +prometheus_server_remote_write_receiver: no + +# prometheus_server_remote_write_destinations: +# example: +# url: "https://mon.example.com/prometheus/api/v1/write" +# basic_auth: +# username: remote +# password_file: /etc/prometheus/prometheus-remote.secret +# tls_config: +# ca: | +# -----BEGIN CERTIFICATE----- +# ... +# -----END CERTIFICATE----- +# write_relabel_configs: +# - source_labels: ['__name__'] +# regex: 'go_gc_.*' +# action: 'drop' +# - source_labels: ['job'] +# regex: 'alertmanager' +# action: 'drop' + # prometheus_server_secret_files: # user: secret +# remote: othersecret diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_coredns.yml b/roles/monitoring/prometheus/server/defaults/main/rules_coredns.yml new file mode 100644 index 00000000..126a7ba4 --- /dev/null +++ b/roles/monitoring/prometheus/server/defaults/main/rules_coredns.yml @@ -0,0 +1,29 @@ +--- +prometheus_server_rules_coredns_extra: [] +prometheus_server_rules_coredns: + - alert: CorednsPanicCount + expr: increase(coredns_panics_total[15m]) > 0 + for: 0m + labels: + severity: critical + annotations: + summary: CoreDNS Panic (instance {{ '{{' }} $labels.instance {{ '}}' }}) + description: "Number of CoreDNS panics encountered has been increasing in the last 15 minutes\n VALUE = {{ '{{' }} $value {{ '}}' }}\n LABELS = {{ '{{' }} $labels {{ '}}' }}" + + - alert: CorednsFailedReloadCount + expr: increase(coredns_reload_failed_total[15m]) > 0 + for: 0m + labels: + severity: critical + annotations: + summary: CoreDNS reload failed (instance {{ '{{' }} $labels.instance {{ '}}' }}) + description: "Number of CoreDNS failed reloads has been increasing in the last 15 minutes\n VALUE = {{ '{{' }} $value {{ '}}' }}\n LABELS = {{ '{{' }} $labels {{ '}}' }}" + + - alert: CorednsBrokenForwardHealthchecks + expr: increase(coredns_forward_healthcheck_broken_total[15m]) > 0 + for: 0m + labels: + severity: warning + annotations: + summary: CoreDNS broken forward healthchecks (instance {{ '{{' }} $labels.instance {{ '}}' }}) + description: "Number of CoreDNS broken forward healthchecks has been increasing in the last 15 minutes\n VALUE = {{ '{{' }} $value {{ '}}' }}\n LABELS = {{ '{{' }} $labels {{ '}}' }}" diff --git a/roles/monitoring/prometheus/server/templates/prometheus.service.j2 b/roles/monitoring/prometheus/server/templates/prometheus.service.j2 index e65e9425..86c30cbd 100644 --- a/roles/monitoring/prometheus/server/templates/prometheus.service.j2 +++ b/roles/monitoring/prometheus/server/templates/prometheus.service.j2 @@ -6,7 +6,7 @@ After=time-sync.target [Service] Restart=on-failure User=prometheus -ExecStart=/usr/bin/prometheus --config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/var/lib/prometheus/metrics2/ --storage.tsdb.retention.time={{ prometheus_server_retention }}{% if prometheus_server_web_external_url is defined %} --web.external-url={{ prometheus_server_web_external_url }}{% endif %}{% if prometheus_server_web_route_prefix is defined %} --web.route-prefix={{ prometheus_server_web_route_prefix }}{% endif %}{% if prometheus_server_auth_users is defined %} --web.config.file=/etc/prometheus/prometheus-web.yml{% endif %} --web.listen-address={{ prometheus_server_web_listen_address }} +ExecStart=/usr/bin/prometheus --config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/var/lib/prometheus/metrics2/ --storage.tsdb.retention.time={{ prometheus_server_retention }}{% if prometheus_server_web_external_url is defined %} --web.external-url={{ prometheus_server_web_external_url }}{% endif %}{% if prometheus_server_web_route_prefix is defined %} --web.route-prefix={{ prometheus_server_web_route_prefix }}{% endif %}{% if prometheus_server_auth_users is defined %} --web.config.file=/etc/prometheus/prometheus-web.yml{% endif %}{% if prometheus_server_remote_write_receiver %} --web.enable-remote-write-receiver{% endif %} --web.listen-address={{ prometheus_server_web_listen_address }} ExecReload=/bin/kill -HUP $MAINPID TimeoutStopSec=20s SendSIGKILL=no diff --git a/roles/monitoring/prometheus/server/templates/prometheus.yml.j2 b/roles/monitoring/prometheus/server/templates/prometheus.yml.j2 index 09d5452d..d72a4815 100644 --- a/roles/monitoring/prometheus/server/templates/prometheus.yml.j2 +++ b/roles/monitoring/prometheus/server/templates/prometheus.yml.j2 @@ -45,7 +45,7 @@ scrape_configs: - targets: ['localhost:9090'] labels: instance: '{{ inventory_hostname }}' -{% if prometheus_server_alertmanager is defined %} +{% if prometheus_server_alertmanager is defined and 'scrape_instance' in prometheus_server_alertmanager %} - job_name: 'alertmanager' {% if 'path_prefix' in prometheus_server_alertmanager %} @@ -58,6 +58,8 @@ scrape_configs: {% endif %} static_configs: - targets: ['{{ prometheus_server_alertmanager.url }}'] + labels: + instance: '{{ prometheus_server_alertmanager.scrape_instance }}' {% endif %} {% for job in (prometheus_server_jobs) %} @@ -94,3 +96,10 @@ scrape_configs: - targets: ['{{ config.url }}'] {% endfor %} {% endfor %} +{% if prometheus_server_remote_write_destinations is defined %} + +remote_write: +{% for name, config in prometheus_server_remote_write_destinations.items() %} + - {{ config | combine({'name': name }) | to_nice_yaml(indent=2) | indent(4) }} +{% endfor %} +{% endif %} |