From e29ce4fdbe2ce669c62777fffa18ae8557e54a73 Mon Sep 17 00:00:00 2001
From: Christian Pointner <equinox@spreadspace.org>
Date: Sun, 30 May 2021 22:28:46 +0200
Subject: prometheus: initial simple server role

---
 inventory/host_vars/ch-mon.yml | 62 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 62 insertions(+)
 create mode 100644 inventory/host_vars/ch-mon.yml

(limited to 'inventory/host_vars')

diff --git a/inventory/host_vars/ch-mon.yml b/inventory/host_vars/ch-mon.yml
new file mode 100644
index 00000000..6bfa58d4
--- /dev/null
+++ b/inventory/host_vars/ch-mon.yml
@@ -0,0 +1,62 @@
+---
+install_jumphost: ch-jump
+
+install:
+  vm:
+    memory: 8G
+    numcpus: 8
+    autostart: yes
+  disks:
+    primary: /dev/sda
+    scsi:
+      sda:
+        type: zfs
+        name: root
+        size: 10g
+      sdb:
+        type: zfs
+        name: data
+        size: 50g
+  interfaces:
+  - bridge: br-svc
+    name: svc0
+  - bridge: br-iot
+    name: iot0
+  - bridge: br-mgmt
+    name: mgmt0
+
+network:
+  nameservers: "{{ network_zones.svc.dns }}"
+  domain: "{{ host_domain }}"
+  systemd_link:
+    interfaces: "{{ install.interfaces }}"
+  primary: &_network_primary_
+    name: svc0
+    address: "{{ network_zones.svc.prefix | ipaddr(network_zones.svc.offsets[inventory_hostname]) | ipaddr('address/prefix') }}"
+    gateway: "{{ network_zones.svc.gateway }}"
+    static_routes:
+    - destination: "{{ network_zones.lan.prefix }}"
+      gateway: "{{ network_zones.svc.prefix | ipaddr(network_zones.svc.offsets['ch-gw-lan']) | ipaddr('address') }}"
+  interfaces:
+  - *_network_primary_
+  - name: iot0
+    address: "{{ network_zones.iot.prefix | ipaddr(network_zones.iot.offsets[inventory_hostname]) | ipaddr('address/prefix') }}"
+  - name: mgmt0
+    address: "{{ network_zones.mgmt.prefix | ipaddr(network_zones.mgmt.offsets[inventory_hostname]) | ipaddr('address/prefix') }}"
+
+
+lvm_groups:
+  mondata:
+    pvs:
+    - /dev/sdb
+
+
+spreadspace_apt_repo_components:
+  - prometheus
+
+prometheus_server_storage:
+  type: lvm
+  vg: mondata
+  lv: prometheus
+  size: 30G
+  fs: ext4
-- 
cgit v1.2.3


From 8bcf938a7b95536c66a34b043915615df489f243 Mon Sep 17 00:00:00 2001
From: Christian Pointner <equinox@spreadspace.org>
Date: Thu, 3 Jun 2021 22:55:21 +0200
Subject: prometheus: fix blackbox exporter icmp probes

---
 inventory/host_vars/ch-mon.yml                     |  4 ++++
 .../prometheus/exporter/blackbox/defaults/main.yml | 10 +++++---
 .../exporter/blackbox/templates/service.j2         |  6 ++++-
 .../prometheus/server/templates/prometheus.yml.j2  | 28 +++++++++++++++++++---
 4 files changed, 41 insertions(+), 7 deletions(-)

(limited to 'inventory/host_vars')

diff --git a/inventory/host_vars/ch-mon.yml b/inventory/host_vars/ch-mon.yml
index 6bfa58d4..222b0e08 100644
--- a/inventory/host_vars/ch-mon.yml
+++ b/inventory/host_vars/ch-mon.yml
@@ -60,3 +60,7 @@ prometheus_server_storage:
   lv: prometheus
   size: 30G
   fs: ext4
+
+prometheus_exporter_blackbox_modules_extra:
+  icmp:
+    prober: icmp
diff --git a/roles/monitoring/prometheus/exporter/blackbox/defaults/main.yml b/roles/monitoring/prometheus/exporter/blackbox/defaults/main.yml
index fcf66555..4e7d8d9a 100644
--- a/roles/monitoring/prometheus/exporter/blackbox/defaults/main.yml
+++ b/roles/monitoring/prometheus/exporter/blackbox/defaults/main.yml
@@ -1,10 +1,8 @@
 ---
 prometheus_exporter_blackbox_modules:
-  icmp:
-    prober: icmp
   tcp_connect:
     prober: tcp
-  tcp_tls:
+  tcp_tls_connect:
     prober: tcp
     tcp:
       tls: true
@@ -12,6 +10,12 @@ prometheus_exporter_blackbox_modules:
         insecure_skip_verify: true
   http_2xx:
     prober: http
+  http_tls_2xx:
+    prober: http
+    http:
+      fail_if_not_ssl: true
+      tls_config:
+        insecure_skip_verify: true
   ssh_banner:
     prober: tcp
     tcp:
diff --git a/roles/monitoring/prometheus/exporter/blackbox/templates/service.j2 b/roles/monitoring/prometheus/exporter/blackbox/templates/service.j2
index c9c5712c..a8a91d0b 100644
--- a/roles/monitoring/prometheus/exporter/blackbox/templates/service.j2
+++ b/roles/monitoring/prometheus/exporter/blackbox/templates/service.j2
@@ -8,8 +8,13 @@ ExecStart=/usr/bin/prometheus-blackbox-exporter --web.listen-address="127.0.0.1:
 ExecReload=/bin/kill -HUP $MAINPID
 
 # systemd hardening-options
+{% if prometheus_exporter_blackbox_modules | combine(prometheus_exporter_blackbox_modules_extra) | dict2items | selectattr('value.prober', 'eq', 'icmp') | length > 0 %}
 AmbientCapabilities=CAP_NET_RAW
 CapabilityBoundingSet=CAP_NET_RAW
+{% else %}
+AmbientCapabilities=
+CapabilityBoundingSet=
+{% endif %}
 DeviceAllow=/dev/null rw
 DevicePolicy=strict
 LockPersonality=true
@@ -17,7 +22,6 @@ MemoryDenyWriteExecute=true
 NoNewPrivileges=true
 PrivateDevices=true
 PrivateTmp=true
-PrivateUsers=true
 ProtectControlGroups=true
 ProtectHome=true
 ProtectKernelModules=true
diff --git a/roles/monitoring/prometheus/server/templates/prometheus.yml.j2 b/roles/monitoring/prometheus/server/templates/prometheus.yml.j2
index eb77d6d1..5eb7c570 100644
--- a/roles/monitoring/prometheus/server/templates/prometheus.yml.j2
+++ b/roles/monitoring/prometheus/server/templates/prometheus.yml.j2
@@ -54,12 +54,12 @@ scrape_configs:
       - target_label: __address__
         replacement: 192.168.32.230:9999
 
-  - job_name: 'tcp_tls'
+  - job_name: 'https'
     metrics_path: /proxy
     params:
       module:
       - blackbox
-      - tcp_tls
+      - http_tls_2xx
     scheme: https
     tls_config:
       ca_file: /etc/ssl/prometheus/ca-crt.pem
@@ -67,7 +67,7 @@ scrape_configs:
       key_file: /etc/ssl/prometheus/server/exporter-key.pem
     static_configs:
       - targets:
-        - web.chaos-at-home.org:443
+        - web.chaos-at-home.org
     relabel_configs:
       - source_labels: [__address__]
         target_label: __param_target
@@ -75,3 +75,25 @@ scrape_configs:
         target_label: instance
       - target_label: __address__
         replacement: 192.168.32.230:9999
+
+  - job_name: 'ssh'
+    metrics_path: /proxy
+    params:
+      module:
+      - blackbox
+      - ssh_banner
+    scheme: https
+    tls_config:
+      ca_file: /etc/ssl/prometheus/ca-crt.pem
+      cert_file: /etc/ssl/prometheus/server/exporter-crt.pem
+      key_file: /etc/ssl/prometheus/server/exporter-key.pem
+    static_configs:
+      - targets:
+        - 192.168.32.230:222
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - target_label: instance
+        replacement: 'ch-mon'
+      - target_label: __address__
+        replacement: 192.168.32.230:9999
-- 
cgit v1.2.3


From 6082a92fa86d121d3ea4256859ee4c9d412e78c0 Mon Sep 17 00:00:00 2001
From: Christian Pointner <equinox@spreadspace.org>
Date: Thu, 10 Jun 2021 01:15:32 +0200
Subject: promethues: remote certificate signing for exporter/base

---
 chaos-at-home/ch-testvm-prometheus.yml             |  7 +++-
 inventory/host_vars/ch-testvm-prometheus.yml       |  3 ++
 roles/monitoring/prometheus/ca/tasks/main.yml      |  2 +-
 .../prometheus/exporter/base/tasks/tls.yml         | 49 +++++++++++++++++++---
 roles/monitoring/prometheus/server/tasks/tls.yml   | 34 ++++++++++-----
 .../prometheus/server/templates/prometheus.yml.j2  | 16 +++----
 6 files changed, 85 insertions(+), 26 deletions(-)

(limited to 'inventory/host_vars')

diff --git a/chaos-at-home/ch-testvm-prometheus.yml b/chaos-at-home/ch-testvm-prometheus.yml
index a34d58e3..9caa2f9a 100644
--- a/chaos-at-home/ch-testvm-prometheus.yml
+++ b/chaos-at-home/ch-testvm-prometheus.yml
@@ -7,5 +7,8 @@
   - role: core/sshd/base
   - role: core/zsh
   - role: core/ntp
-  - role: kubernetes/base
-  - role: kubernetes/standalone/base
+  - role: apt-repo/spreadspace
+  - role: monitoring/prometheus/exporter/base
+  - role: monitoring/prometheus/exporter/node
+  # - role: kubernetes/base
+  # - role: kubernetes/standalone/base
diff --git a/inventory/host_vars/ch-testvm-prometheus.yml b/inventory/host_vars/ch-testvm-prometheus.yml
index d11d565c..e539735f 100644
--- a/inventory/host_vars/ch-testvm-prometheus.yml
+++ b/inventory/host_vars/ch-testvm-prometheus.yml
@@ -33,6 +33,9 @@ network:
   - *_network_primary_
 
 
+spreadspace_apt_repo_components:
+  - prometheus
+
 
 containerd_storage:
   type: lvm
diff --git a/roles/monitoring/prometheus/ca/tasks/main.yml b/roles/monitoring/prometheus/ca/tasks/main.yml
index 9f166321..cde4a267 100644
--- a/roles/monitoring/prometheus/ca/tasks/main.yml
+++ b/roles/monitoring/prometheus/ca/tasks/main.yml
@@ -34,7 +34,6 @@
     useCommonNameForSAN: no
     key_usage:
     - cRLSign
-    - digitalSignature
     - keyCertSign
     key_usage_critical: yes
     basic_constraints:
@@ -50,3 +49,4 @@
     provider: selfsigned
     selfsigned_digest: sha256
     selfsigned_not_after: "+18250d" ## 50 years
+    selfsigned_create_subject_key_identifier: always_create
diff --git a/roles/monitoring/prometheus/exporter/base/tasks/tls.yml b/roles/monitoring/prometheus/exporter/base/tasks/tls.yml
index b2731b09..72186acb 100644
--- a/roles/monitoring/prometheus/exporter/base/tasks/tls.yml
+++ b/roles/monitoring/prometheus/exporter/base/tasks/tls.yml
@@ -45,17 +45,56 @@
     - 'CA:FALSE'
     basic_constraints_critical: yes
 
-## TODO: implement remote singing using server
+- name: slurp CSR
+  slurp:
+    src: /etc/ssl/prometheus/exporter/csr.pem
+  register: prometheus_exporter_server_csr
 
-- name: create exporter certificate
-  openssl_certificate:
+- name: check if exporter certificate exists
+  stat:
     path: /etc/ssl/prometheus/exporter/crt.pem
-    csr_path: /etc/ssl/prometheus/exporter/csr.pem
+  register: prometheus_exporter_server_cert
+
+- name: read exporter client certificate issuer key id and validity
+  when: prometheus_exporter_server_cert.stat.exists
+  openssl_certificate_info:
+    path: /etc/ssl/prometheus/exporter/crt.pem
+    valid_at:
+      ten_years: '+3650d'
+  register: prometheus_exporter_server_cert_info
+
+- name: slurp existing exporter certificate
+  when: prometheus_exporter_server_cert.stat.exists
+  slurp:
+    src: /etc/ssl/prometheus/exporter/crt.pem
+  register: prometheus_exporter_server_cert_current
+
+- name: generate exporter certificate
+  delegate_to: "{{ promethues_server }}"
+  community.crypto.x509_certificate_pipe:
+    content: "{{ prometheus_exporter_server_cert_current.content | default('') | b64decode }}"
+    csr_content: "{{ prometheus_exporter_server_csr.content | b64decode }}"
     provider: ownca
     ownca_path: /etc/ssl/prometheus/ca-crt.pem
     ownca_privatekey_path: /etc/ssl/prometheus/ca/key.pem
     ownca_digest: sha256
     ownca_not_after: "+18250d" ## 50 years
+    force: "{{ prometheus_exporter_server_cert.stat.exists and (not prometheus_exporter_server_cert_info.valid_at.ten_years) }}"
+  register: prometheus_exporter_server_cert
+
+- name: store exporter certificate
+  copy:
+    content: "{{ prometheus_exporter_server_cert.certificate }}"
+    dest: /etc/ssl/prometheus/exporter/crt.pem
   notify: restart prometheus-exporter-exporter
 
-## TODO: install /etc/ssl/prometheus/ca-crt.pem from server
+- name: slurp CA certificate
+  delegate_to: "{{ promethues_server }}"
+  slurp:
+    src: /etc/ssl/prometheus/ca-crt.pem
+  register: prometheus_exporter_ca_certificate
+
+- name: install CA certificate
+  copy:
+    content: "{{ prometheus_exporter_ca_certificate.content | b64decode }}"
+    dest: /etc/ssl/prometheus/ca-crt.pem
diff --git a/roles/monitoring/prometheus/server/tasks/tls.yml b/roles/monitoring/prometheus/server/tasks/tls.yml
index 5c112e12..940c69b1 100644
--- a/roles/monitoring/prometheus/server/tasks/tls.yml
+++ b/roles/monitoring/prometheus/server/tasks/tls.yml
@@ -17,9 +17,9 @@
     group: prometheus
     mode: 0750
 
-- name: create private key to connect to exporter
+- name: create private key for scrape-client certificate
   openssl_privatekey:
-    path: /etc/ssl/prometheus/server/exporter-key.pem
+    path: /etc/ssl/prometheus/server/scrape-key.pem
     type: RSA
     size: 4096
     owner: prometheus
@@ -27,10 +27,10 @@
     mode: 0400
   notify: reload prometheus
 
-- name: create signing request for client certificate to connect to exporter
+- name: create signing request for scrape-client certificate
   openssl_csr:
-    path: /etc/ssl/prometheus/server/exporter-csr.pem
-    privatekey_path: /etc/ssl/prometheus/server/exporter-key.pem
+    path: /etc/ssl/prometheus/server/scrape-csr.pem
+    privatekey_path: /etc/ssl/prometheus/server/scrape-key.pem
     CN: "{{ inventory_hostname }}"
     subject_alt_name:
     - "DNS:{{ host_name }}.{{ host_domain }}"
@@ -45,17 +45,31 @@
     - 'CA:FALSE'
     basic_constraints_critical: yes
 
+## TODO: install /etc/ssl/prometheus/ca-crt.pem from CA host
+
+- name: check if scrape-client certificate exists
+  stat:
+    path: /etc/ssl/prometheus/server/scrape-crt.pem
+  register: prometheus_server_scrape_client_cert
+
+- name: check scrape-client certificate validity
+  when: prometheus_server_scrape_client_cert.stat.exists
+  openssl_certificate_info:
+    path: /etc/ssl/prometheus/server/scrape-crt.pem
+    valid_at:
+      ten_years: '+3650d'
+  register: prometheus_server_scrape_client_cert_info
+
 ## TODO: implement remote signing?
 
-- name: create client certificate to connect to exporter
+- name: create scrape-client certificate
   openssl_certificate:
-    path: /etc/ssl/prometheus/server/exporter-crt.pem
-    csr_path: /etc/ssl/prometheus/server/exporter-csr.pem
+    path: /etc/ssl/prometheus/server/scrape-crt.pem
+    csr_path: /etc/ssl/prometheus/server/scrape-csr.pem
     provider: ownca
     ownca_path: /etc/ssl/prometheus/ca-crt.pem
     ownca_privatekey_path: /etc/ssl/prometheus/ca/key.pem
     ownca_digest: sha256
     ownca_not_after: "+18250d" ## 50 years
+    force: "{{ prometheus_server_scrape_client_cert.stat.exists and (not prometheus_server_scrape_client_cert_info.valid_at.ten_years) }}"
   notify: reload prometheus
-
-## TODO: install /etc/ssl/prometheus/ca-crt.pem from server
diff --git a/roles/monitoring/prometheus/server/templates/prometheus.yml.j2 b/roles/monitoring/prometheus/server/templates/prometheus.yml.j2
index 5eb7c570..3975c74d 100644
--- a/roles/monitoring/prometheus/server/templates/prometheus.yml.j2
+++ b/roles/monitoring/prometheus/server/templates/prometheus.yml.j2
@@ -23,8 +23,8 @@ scrape_configs:
     scheme: https
     tls_config:
       ca_file: /etc/ssl/prometheus/ca-crt.pem
-      cert_file: /etc/ssl/prometheus/server/exporter-crt.pem
-      key_file: /etc/ssl/prometheus/server/exporter-key.pem
+      cert_file: /etc/ssl/prometheus/server/scrape-crt.pem
+      key_file: /etc/ssl/prometheus/server/scrape-key.pem
     file_sd_configs:
     - files:
       - "/etc/prometheus/jobs/{{ job }}/*.yml"
@@ -40,8 +40,8 @@ scrape_configs:
     scheme: https
     tls_config:
       ca_file: /etc/ssl/prometheus/ca-crt.pem
-      cert_file: /etc/ssl/prometheus/server/exporter-crt.pem
-      key_file: /etc/ssl/prometheus/server/exporter-key.pem
+      cert_file: /etc/ssl/prometheus/server/scrape-crt.pem
+      key_file: /etc/ssl/prometheus/server/scrape-key.pem
     static_configs:
       - targets:
         - 62.99.185.129
@@ -63,8 +63,8 @@ scrape_configs:
     scheme: https
     tls_config:
       ca_file: /etc/ssl/prometheus/ca-crt.pem
-      cert_file: /etc/ssl/prometheus/server/exporter-crt.pem
-      key_file: /etc/ssl/prometheus/server/exporter-key.pem
+      cert_file: /etc/ssl/prometheus/server/scrape-crt.pem
+      key_file: /etc/ssl/prometheus/server/scrape-key.pem
     static_configs:
       - targets:
         - web.chaos-at-home.org
@@ -85,8 +85,8 @@ scrape_configs:
     scheme: https
     tls_config:
       ca_file: /etc/ssl/prometheus/ca-crt.pem
-      cert_file: /etc/ssl/prometheus/server/exporter-crt.pem
-      key_file: /etc/ssl/prometheus/server/exporter-key.pem
+      cert_file: /etc/ssl/prometheus/server/scrape-crt.pem
+      key_file: /etc/ssl/prometheus/server/scrape-key.pem
     static_configs:
       - targets:
         - 192.168.32.230:222
-- 
cgit v1.2.3


From d0482708def7d7b5165590db30bdca014d187528 Mon Sep 17 00:00:00 2001
From: Christian Pointner <equinox@spreadspace.org>
Date: Sun, 13 Jun 2021 19:23:46 +0200
Subject: add meta dep for prometheus exporter

---
 chaos-at-home/ch-mon.yml                               | 4 +---
 chaos-at-home/ch-testvm-prometheus.yml                 | 3 +--
 inventory/host_vars/ch-mon.yml                         | 4 ++++
 roles/monitoring/prometheus/exporter/defaults/main.yml | 3 +++
 roles/monitoring/prometheus/exporter/meta/main.yml     | 7 +++++++
 5 files changed, 16 insertions(+), 5 deletions(-)
 create mode 100644 roles/monitoring/prometheus/exporter/defaults/main.yml
 create mode 100644 roles/monitoring/prometheus/exporter/meta/main.yml

(limited to 'inventory/host_vars')

diff --git a/chaos-at-home/ch-mon.yml b/chaos-at-home/ch-mon.yml
index bce4adab..248de5d6 100644
--- a/chaos-at-home/ch-mon.yml
+++ b/chaos-at-home/ch-mon.yml
@@ -10,7 +10,5 @@
   - role: storage/lvm/groups
   - role: apt-repo/spreadspace
   - role: monitoring/prometheus/ca
-  - role: monitoring/prometheus/exporter/base
-  - role: monitoring/prometheus/exporter/node
-  - role: monitoring/prometheus/exporter/blackbox
+  - role: monitoring/prometheus/exporter
   - role: monitoring/prometheus/server
diff --git a/chaos-at-home/ch-testvm-prometheus.yml b/chaos-at-home/ch-testvm-prometheus.yml
index 9caa2f9a..3fd99d41 100644
--- a/chaos-at-home/ch-testvm-prometheus.yml
+++ b/chaos-at-home/ch-testvm-prometheus.yml
@@ -8,7 +8,6 @@
   - role: core/zsh
   - role: core/ntp
   - role: apt-repo/spreadspace
-  - role: monitoring/prometheus/exporter/base
-  - role: monitoring/prometheus/exporter/node
+  - role: monitoring/prometheus/exporter
   # - role: kubernetes/base
   # - role: kubernetes/standalone/base
diff --git a/inventory/host_vars/ch-mon.yml b/inventory/host_vars/ch-mon.yml
index 222b0e08..25dae3ac 100644
--- a/inventory/host_vars/ch-mon.yml
+++ b/inventory/host_vars/ch-mon.yml
@@ -61,6 +61,10 @@ prometheus_server_storage:
   size: 30G
   fs: ext4
 
+prometheus_exporters:
+  - node
+  - blackbox
+
 prometheus_exporter_blackbox_modules_extra:
   icmp:
     prober: icmp
diff --git a/roles/monitoring/prometheus/exporter/defaults/main.yml b/roles/monitoring/prometheus/exporter/defaults/main.yml
new file mode 100644
index 00000000..858c1837
--- /dev/null
+++ b/roles/monitoring/prometheus/exporter/defaults/main.yml
@@ -0,0 +1,3 @@
+---
+prometheus_exporters:
+  - node
diff --git a/roles/monitoring/prometheus/exporter/meta/main.yml b/roles/monitoring/prometheus/exporter/meta/main.yml
new file mode 100644
index 00000000..ddb30f9a
--- /dev/null
+++ b/roles/monitoring/prometheus/exporter/meta/main.yml
@@ -0,0 +1,7 @@
+---
+dependencies:
+  - role: monitoring/prometheus/exporter/base
+  - role: monitoring/prometheus/exporter/node
+    when: "'node' in prometheus_exporters"
+  - role: monitoring/prometheus/exporter/blackbox
+    when: "'blackbox' in prometheus_exporters"
-- 
cgit v1.2.3


From 4e5f835b6dd5aee26a663155211ee5dd3642d07d Mon Sep 17 00:00:00 2001
From: Christian Pointner <equinox@spreadspace.org>
Date: Mon, 14 Jun 2021 00:49:48 +0200
Subject: make prometheus exporter list groupvars

---
 inventory/group_vars/promzone-chaos-at-home/vars.yml   | 4 ++++
 inventory/host_vars/ch-mon.yml                         | 3 +--
 roles/monitoring/prometheus/exporter/defaults/main.yml | 3 ---
 roles/monitoring/prometheus/exporter/meta/main.yml     | 4 ++--
 4 files changed, 7 insertions(+), 7 deletions(-)
 delete mode 100644 roles/monitoring/prometheus/exporter/defaults/main.yml

(limited to 'inventory/host_vars')

diff --git a/inventory/group_vars/promzone-chaos-at-home/vars.yml b/inventory/group_vars/promzone-chaos-at-home/vars.yml
index 8a0d0aa8..2345292b 100644
--- a/inventory/group_vars/promzone-chaos-at-home/vars.yml
+++ b/inventory/group_vars/promzone-chaos-at-home/vars.yml
@@ -3,3 +3,7 @@ promethues_server: ch-mon
 promethues_zone_name: chaos@home
 
 prometheus_zone_targets: "{{ groups['promzone-chaos-at-home'] }}"
+
+prometheus_exporters_extra: []
+prometheus_exporters_default:
+  - node
diff --git a/inventory/host_vars/ch-mon.yml b/inventory/host_vars/ch-mon.yml
index 25dae3ac..025289a4 100644
--- a/inventory/host_vars/ch-mon.yml
+++ b/inventory/host_vars/ch-mon.yml
@@ -61,8 +61,7 @@ prometheus_server_storage:
   size: 30G
   fs: ext4
 
-prometheus_exporters:
-  - node
+prometheus_exporters_extra:
   - blackbox
 
 prometheus_exporter_blackbox_modules_extra:
diff --git a/roles/monitoring/prometheus/exporter/defaults/main.yml b/roles/monitoring/prometheus/exporter/defaults/main.yml
deleted file mode 100644
index 858c1837..00000000
--- a/roles/monitoring/prometheus/exporter/defaults/main.yml
+++ /dev/null
@@ -1,3 +0,0 @@
----
-prometheus_exporters:
-  - node
diff --git a/roles/monitoring/prometheus/exporter/meta/main.yml b/roles/monitoring/prometheus/exporter/meta/main.yml
index ddb30f9a..d1d3eac7 100644
--- a/roles/monitoring/prometheus/exporter/meta/main.yml
+++ b/roles/monitoring/prometheus/exporter/meta/main.yml
@@ -2,6 +2,6 @@
 dependencies:
   - role: monitoring/prometheus/exporter/base
   - role: monitoring/prometheus/exporter/node
-    when: "'node' in prometheus_exporters"
+    when: "'node' in (prometheus_exporters_default | union(prometheus_exporters_extra))"
   - role: monitoring/prometheus/exporter/blackbox
-    when: "'blackbox' in prometheus_exporters"
+    when: "'blackbox' in (prometheus_exporters_default | union(prometheus_exporters_extra))"
-- 
cgit v1.2.3


From 4ab6efde0658c8998b92bb565d3660d5478c9406 Mon Sep 17 00:00:00 2001
From: Christian Pointner <equinox@spreadspace.org>
Date: Mon, 21 Jun 2021 20:10:43 +0200
Subject: install grafana on ch-mon

---
 chaos-at-home/ch-mon.yml           |  2 ++
 chaos-at-home/host_vars/ch-mon.yml | 10 ++++++++++
 inventory/host_vars/ch-mon.yml     |  3 +++
 3 files changed, 15 insertions(+)
 create mode 100644 chaos-at-home/host_vars/ch-mon.yml

(limited to 'inventory/host_vars')

diff --git a/chaos-at-home/ch-mon.yml b/chaos-at-home/ch-mon.yml
index 248de5d6..8e25d6ec 100644
--- a/chaos-at-home/ch-mon.yml
+++ b/chaos-at-home/ch-mon.yml
@@ -8,7 +8,9 @@
   - role: core/zsh
   - role: core/ntp
   - role: storage/lvm/groups
+  - role: nginx/base
   - role: apt-repo/spreadspace
   - role: monitoring/prometheus/ca
   - role: monitoring/prometheus/exporter
   - role: monitoring/prometheus/server
+  - role: monitoring/grafana
diff --git a/chaos-at-home/host_vars/ch-mon.yml b/chaos-at-home/host_vars/ch-mon.yml
new file mode 100644
index 00000000..02b3883a
--- /dev/null
+++ b/chaos-at-home/host_vars/ch-mon.yml
@@ -0,0 +1,10 @@
+$ANSIBLE_VAULT;1.2;AES256;chaos-at-home
+64303062373862353734336666336331613033343966353530323764303830386162633039656461
+3630326139303239353862336338306662646230663332660a646139363333376332653331376135
+35366465323236396234396133653364343130383631366232646362363930343938306438613161
+3965303365613234380a626232376239303165313536653439353136643861646631323031313837
+32373737326539646336373661376539336663346637616662313133663663313733353538636435
+31366638616632333836656561363464353635336638343436386339353065393530376531353039
+63343336383732336533333336303766323839646636643235313463306436353066653261393136
+64336263383336653765343335613038633263306638336639653230346633366539613431616434
+3733
diff --git a/inventory/host_vars/ch-mon.yml b/inventory/host_vars/ch-mon.yml
index 025289a4..6e064764 100644
--- a/inventory/host_vars/ch-mon.yml
+++ b/inventory/host_vars/ch-mon.yml
@@ -67,3 +67,6 @@ prometheus_exporters_extra:
 prometheus_exporter_blackbox_modules_extra:
   icmp:
     prober: icmp
+
+
+grafana_secret_key: "{{ vault_grafana_secret_key }}"
-- 
cgit v1.2.3


From 7440787a0cf4dd2bab4439ba481e34ead78c0c55 Mon Sep 17 00:00:00 2001
From: Christian Pointner <equinox@spreadspace.org>
Date: Mon, 21 Jun 2021 23:18:49 +0200
Subject: grafana: smtp settings

---
 inventory/host_vars/ch-mon.yml             |  6 ++++++
 roles/monitoring/grafana/defaults/main.yml |  2 ++
 roles/monitoring/grafana/tasks/main.yml    | 13 ++++++++++++-
 3 files changed, 20 insertions(+), 1 deletion(-)

(limited to 'inventory/host_vars')

diff --git a/inventory/host_vars/ch-mon.yml b/inventory/host_vars/ch-mon.yml
index 6e064764..a889780d 100644
--- a/inventory/host_vars/ch-mon.yml
+++ b/inventory/host_vars/ch-mon.yml
@@ -70,3 +70,9 @@ prometheus_exporter_blackbox_modules_extra:
 
 
 grafana_secret_key: "{{ vault_grafana_secret_key }}"
+
+grafana_config_smtp:
+   enabled: true
+   host: "{{ network_zones.lan.prefix | ipaddr(network_zones.lan.offsets['ch-prometheus-legacy']) | ipaddr('address') }}:25"
+   from_name: "chaos@home Grafana"
+   from_address: noreply@chaos-at-home.org
diff --git a/roles/monitoring/grafana/defaults/main.yml b/roles/monitoring/grafana/defaults/main.yml
index 8798dfb5..7141d488 100644
--- a/roles/monitoring/grafana/defaults/main.yml
+++ b/roles/monitoring/grafana/defaults/main.yml
@@ -19,3 +19,5 @@ grafana_config_security:
 grafana_config_users:
   allow_sign_up: false
   allow_org_create: false
+
+grafana_config_smtp: {}
diff --git a/roles/monitoring/grafana/tasks/main.yml b/roles/monitoring/grafana/tasks/main.yml
index 2e7594ec..0cf968f1 100644
--- a/roles/monitoring/grafana/tasks/main.yml
+++ b/roles/monitoring/grafana/tasks/main.yml
@@ -46,12 +46,23 @@
   loop_control:
     label: "{{ item.key }}"
   ini_file:
-    path: /etc/grafana/grafana.inig
+    path: /etc/grafana/grafana.ini
     section: users
     option: "{{ item.key }}"
     value: "{{ item.value | string }}"
   notify: restart grafana
 
+- name: configure grafana smtp
+  loop: "{{ grafana_config_smtp | dict2items }}"
+  loop_control:
+    label: "{{ item.key }}"
+  ini_file:
+    path: /etc/grafana/grafana.ini
+    section: smtp
+    option: "{{ item.key }}"
+    value: "{{ item.value | string }}"
+  notify: restart grafana
+
 - name: make sure grafan-server is enabled and started
   systemd:
     name: grafana-server
-- 
cgit v1.2.3


From 8e9b9ef4e15084113d833b731aee485d0c989e16 Mon Sep 17 00:00:00 2001
From: Christian Pointner <equinox@spreadspace.org>
Date: Mon, 21 Jun 2021 23:58:37 +0200
Subject: some more grafana configs

---
 inventory/host_vars/ch-mon.yml             | 11 +++++++----
 roles/monitoring/grafana/defaults/main.yml |  1 +
 roles/monitoring/grafana/tasks/main.yml    | 11 +++++++++++
 3 files changed, 19 insertions(+), 4 deletions(-)

(limited to 'inventory/host_vars')

diff --git a/inventory/host_vars/ch-mon.yml b/inventory/host_vars/ch-mon.yml
index a889780d..03a9b80a 100644
--- a/inventory/host_vars/ch-mon.yml
+++ b/inventory/host_vars/ch-mon.yml
@@ -72,7 +72,10 @@ prometheus_exporter_blackbox_modules_extra:
 grafana_secret_key: "{{ vault_grafana_secret_key }}"
 
 grafana_config_smtp:
-   enabled: true
-   host: "{{ network_zones.lan.prefix | ipaddr(network_zones.lan.offsets['ch-prometheus-legacy']) | ipaddr('address') }}:25"
-   from_name: "chaos@home Grafana"
-   from_address: noreply@chaos-at-home.org
+  enabled: true
+  host: "{{ network_zones.lan.prefix | ipaddr(network_zones.lan.offsets['ch-prometheus-legacy']) | ipaddr('address') }}:25"
+  from_name: "chaos@home Grafana"
+  from_address: noreply@chaos-at-home.org
+
+grafana_config_plugins:
+  enable_alpha: true
diff --git a/roles/monitoring/grafana/defaults/main.yml b/roles/monitoring/grafana/defaults/main.yml
index 7141d488..0118b8cb 100644
--- a/roles/monitoring/grafana/defaults/main.yml
+++ b/roles/monitoring/grafana/defaults/main.yml
@@ -21,3 +21,4 @@ grafana_config_users:
   allow_org_create: false
 
 grafana_config_smtp: {}
+grafana_config_plugins: {}
diff --git a/roles/monitoring/grafana/tasks/main.yml b/roles/monitoring/grafana/tasks/main.yml
index 0cf968f1..55cce412 100644
--- a/roles/monitoring/grafana/tasks/main.yml
+++ b/roles/monitoring/grafana/tasks/main.yml
@@ -63,6 +63,17 @@
     value: "{{ item.value | string }}"
   notify: restart grafana
 
+- name: configure grafana plugins
+  loop: "{{ grafana_config_plugins | dict2items }}"
+  loop_control:
+    label: "{{ item.key }}"
+  ini_file:
+    path: /etc/grafana/grafana.ini
+    section: plugins
+    option: "{{ item.key }}"
+    value: "{{ item.value | string }}"
+  notify: restart grafana
+
 - name: make sure grafan-server is enabled and started
   systemd:
     name: grafana-server
-- 
cgit v1.2.3


From 5408325a13337672ea09907278ff97b42de60b36 Mon Sep 17 00:00:00 2001
From: Christian Pointner <equinox@spreadspace.org>
Date: Tue, 22 Jun 2021 23:29:18 +0200
Subject: add minimalistic role for prometheus/alertmanager

---
 chaos-at-home/ch-mon.yml                           |  1 +
 inventory/host_vars/ch-mon.yml                     |  7 +++-
 .../prometheus/alertmanager/defaults/main.yml      |  5 +++
 .../prometheus/alertmanager/handlers/main.yml      | 10 +++++
 .../prometheus/alertmanager/tasks/main.yml         | 45 ++++++++++++++++++++++
 .../alertmanager/templates/alertmanager.yml.j2     | 17 ++++++++
 .../templates/prometheus-alertmanager.service.j2   | 37 ++++++++++++++++++
 7 files changed, 121 insertions(+), 1 deletion(-)
 create mode 100644 roles/monitoring/prometheus/alertmanager/defaults/main.yml
 create mode 100644 roles/monitoring/prometheus/alertmanager/handlers/main.yml
 create mode 100644 roles/monitoring/prometheus/alertmanager/tasks/main.yml
 create mode 100644 roles/monitoring/prometheus/alertmanager/templates/alertmanager.yml.j2
 create mode 100644 roles/monitoring/prometheus/alertmanager/templates/prometheus-alertmanager.service.j2

(limited to 'inventory/host_vars')

diff --git a/chaos-at-home/ch-mon.yml b/chaos-at-home/ch-mon.yml
index 8e25d6ec..906e8adc 100644
--- a/chaos-at-home/ch-mon.yml
+++ b/chaos-at-home/ch-mon.yml
@@ -12,5 +12,6 @@
   - role: apt-repo/spreadspace
   - role: monitoring/prometheus/ca
   - role: monitoring/prometheus/exporter
+  - role: monitoring/prometheus/alertmanager
   - role: monitoring/prometheus/server
   - role: monitoring/grafana
diff --git a/inventory/host_vars/ch-mon.yml b/inventory/host_vars/ch-mon.yml
index 03a9b80a..c0551768 100644
--- a/inventory/host_vars/ch-mon.yml
+++ b/inventory/host_vars/ch-mon.yml
@@ -68,6 +68,11 @@ prometheus_exporter_blackbox_modules_extra:
   icmp:
     prober: icmp
 
+promethues_alertmanager_smtp:
+  smarthost: "{{ network_zones.lan.prefix | ipaddr(network_zones.lan.offsets['ch-prometheus-legacy']) | ipaddr('address') }}:25"
+  from: "noreply@chaos-at-home.org"
+  require_tls: no
+
 
 grafana_secret_key: "{{ vault_grafana_secret_key }}"
 
@@ -75,7 +80,7 @@ grafana_config_smtp:
   enabled: true
   host: "{{ network_zones.lan.prefix | ipaddr(network_zones.lan.offsets['ch-prometheus-legacy']) | ipaddr('address') }}:25"
   from_name: "chaos@home Grafana"
-  from_address: noreply@chaos-at-home.org
+  from_address: "noreply@chaos-at-home.org"
 
 grafana_config_plugins:
   enable_alpha: true
diff --git a/roles/monitoring/prometheus/alertmanager/defaults/main.yml b/roles/monitoring/prometheus/alertmanager/defaults/main.yml
new file mode 100644
index 00000000..34b03df0
--- /dev/null
+++ b/roles/monitoring/prometheus/alertmanager/defaults/main.yml
@@ -0,0 +1,5 @@
+---
+promethues_alertmanager_smtp:
+  smarthost: "127.0.0.1:25"
+  from: "noreply@example.com"
+  require_tls: no
diff --git a/roles/monitoring/prometheus/alertmanager/handlers/main.yml b/roles/monitoring/prometheus/alertmanager/handlers/main.yml
new file mode 100644
index 00000000..571b1f7c
--- /dev/null
+++ b/roles/monitoring/prometheus/alertmanager/handlers/main.yml
@@ -0,0 +1,10 @@
+---
+- name: restart prometheus-alertmanager
+  service:
+    name: prometheus-alertmanager
+    state: restarted
+
+- name: reload prometheus-alertmanager
+  service:
+    name: prometheus-alertmanager
+    state: reloaded
diff --git a/roles/monitoring/prometheus/alertmanager/tasks/main.yml b/roles/monitoring/prometheus/alertmanager/tasks/main.yml
new file mode 100644
index 00000000..fe8ce9ca
--- /dev/null
+++ b/roles/monitoring/prometheus/alertmanager/tasks/main.yml
@@ -0,0 +1,45 @@
+---
+- name: check if prometheus apt component of spreadspace repo is enabled
+  assert:
+    msg: "please enable the 'prometheus' component of spreadspace repo using 'spreadspace_apt_repo_components'"
+    that:
+    - spreadspace_apt_repo_components is defined
+    - "'prometheus' in spreadspace_apt_repo_components"
+
+- name: install apt packages
+  apt:
+    name: prom-alertmanager
+    state: present
+
+- name: add user for server
+  user:
+    name: prometheus-alertmanager
+    system: yes
+    home: /nonexistent
+    create_home: no
+
+- name: create data directory
+  file:
+    path: /var/lib/prometheus/alertmanager
+    state: directory
+    owner: prometheus-alertmanager
+    group: prometheus-alertmanager
+
+- name: generate configuration file
+  template:
+    src: alertmanager.yml.j2
+    dest: /etc/prometheus/alertmanager.yml
+  notify: reload prometheus-alertmanager
+
+- name: generate systemd service unit
+  template:
+    src: prometheus-alertmanager.service.j2
+    dest: /etc/systemd/system/prometheus-alertmanager.service
+  notify: restart prometheus-alertmanager
+
+- name: make sure alertmanager is enabled and started
+  systemd:
+    name: prometheus-alertmanager.service
+    daemon_reload: yes
+    state: started
+    enabled: yes
diff --git a/roles/monitoring/prometheus/alertmanager/templates/alertmanager.yml.j2 b/roles/monitoring/prometheus/alertmanager/templates/alertmanager.yml.j2
new file mode 100644
index 00000000..b1d40bb2
--- /dev/null
+++ b/roles/monitoring/prometheus/alertmanager/templates/alertmanager.yml.j2
@@ -0,0 +1,17 @@
+# {{ ansible_managed }}
+
+global:
+  smtp_smarthost: '{{ promethues_alertmanager_smtp.smarthost }}'
+  smtp_from: '{{ promethues_alertmanager_smtp.from }}'
+  smtp_require_tls: {{ promethues_alertmanager_smtp.require_tls | ternary('true', 'false') }}
+
+route:
+  receiver: empty
+
+  routes:
+  - match_re:
+      instance: ^$
+    receiver: empty
+
+receivers:
+- name: empty
diff --git a/roles/monitoring/prometheus/alertmanager/templates/prometheus-alertmanager.service.j2 b/roles/monitoring/prometheus/alertmanager/templates/prometheus-alertmanager.service.j2
new file mode 100644
index 00000000..f290dca8
--- /dev/null
+++ b/roles/monitoring/prometheus/alertmanager/templates/prometheus-alertmanager.service.j2
@@ -0,0 +1,37 @@
+[Unit]
+Description=Alertmanager for Prometheus Monitoring system
+Documentation=https://prometheus.io/docs/alerting/alertmanager/
+
+[Service]
+Restart=on-failure
+User=prometheus-alertmanager
+ExecStart=/usr/bin/prometheus-alertmanager --config.file=/etc/prometheus/alertmanager.yml --cluster.listen-address= --storage.path="/var/lib/prometheus/alertmanager"
+ExecReload=/bin/kill -HUP $MAINPID
+TimeoutStopSec=20s
+SendSIGKILL=no
+
+# systemd hardening-options
+AmbientCapabilities=
+CapabilityBoundingSet=
+DeviceAllow=/dev/null rw
+DevicePolicy=strict
+LimitMEMLOCK=0
+LimitNOFILE=8192
+LockPersonality=true
+MemoryDenyWriteExecute=true
+NoNewPrivileges=true
+PrivateDevices=true
+PrivateTmp=true
+PrivateUsers=true
+ProtectControlGroups=true
+ProtectHome=true
+ProtectKernelModules=true
+ProtectKernelTunables=true
+ProtectSystem=full
+RemoveIPC=true
+RestrictNamespaces=true
+RestrictRealtime=true
+SystemCallArchitectures=native
+
+[Install]
+WantedBy=multi-user.target
-- 
cgit v1.2.3


From 6cf380956bdd31292b4ccf51b1bbc217b93bf45f Mon Sep 17 00:00:00 2001
From: Christian Pointner <equinox@spreadspace.org>
Date: Wed, 23 Jun 2021 23:06:40 +0200
Subject: prometheus: connect server to alertmanager if configured

---
 inventory/host_vars/ch-mon.yml                     |  5 ++-
 .../prometheus/server/defaults/main/main.yml       |  5 ++-
 .../server/defaults/main/rules_prometheus.yml      | 47 ++++++++++++++++++++++
 .../prometheus/server/templates/prometheus.yml.j2  | 13 ++++++
 4 files changed, 68 insertions(+), 2 deletions(-)

(limited to 'inventory/host_vars')

diff --git a/inventory/host_vars/ch-mon.yml b/inventory/host_vars/ch-mon.yml
index c0551768..111ffb55 100644
--- a/inventory/host_vars/ch-mon.yml
+++ b/inventory/host_vars/ch-mon.yml
@@ -61,6 +61,10 @@ prometheus_server_storage:
   size: 30G
   fs: ext4
 
+prometheus_server_alertmanager:
+  url: "127.0.0.1:9093"
+
+
 prometheus_exporters_extra:
   - blackbox
 
@@ -73,7 +77,6 @@ promethues_alertmanager_smtp:
   from: "noreply@chaos-at-home.org"
   require_tls: no
 
-
 grafana_secret_key: "{{ vault_grafana_secret_key }}"
 
 grafana_config_smtp:
diff --git a/roles/monitoring/prometheus/server/defaults/main/main.yml b/roles/monitoring/prometheus/server/defaults/main/main.yml
index b10d6f17..8e7fea4b 100644
--- a/roles/monitoring/prometheus/server/defaults/main/main.yml
+++ b/roles/monitoring/prometheus/server/defaults/main/main.yml
@@ -9,5 +9,8 @@ prometheus_server_jobs:
   - node
 
 prometheus_server_rules:
-  prometheus: "{{ prometheus_server_rules_prometheus + prometheus_server_rules_prometheus_extra }}"
+  prometheus: "{{ prometheus_server_rules_prometheus + ((prometheus_server_alertmanager is defined) | ternary(prometheus_server_rules_prometheus_alertmanager, [])) + prometheus_server_rules_prometheus_extra }}"
   node: "{{ prometheus_server_rules_node + prometheus_server_rules_prometheus_extra }}"
+
+# prometheus_server_alertmanager:
+#   url: "127.0.0.1:9093"
diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_prometheus.yml b/roles/monitoring/prometheus/server/defaults/main/rules_prometheus.yml
index 6d84efa4..8d4672b1 100644
--- a/roles/monitoring/prometheus/server/defaults/main/rules_prometheus.yml
+++ b/roles/monitoring/prometheus/server/defaults/main/rules_prometheus.yml
@@ -190,3 +190,50 @@ prometheus_server_rules_prometheus:
     annotations:
       summary: Prometheus TSDB WAL truncations failed (instance {{ '{{' }} $labels.instance {{ '}}' }})
       description: "Prometheus encountered {{ '{{' }} $value {{ '}}' }} TSDB WAL truncation failures\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+
+prometheus_server_rules_prometheus_alertmanager:
+  - alert: PrometheusAlertmanagerConfigurationReloadFailure
+    expr: alertmanager_config_last_reload_successful != 1
+    for: 0m
+    labels:
+      severity: warning
+    annotations:
+      summary: Prometheus AlertManager configuration reload failure (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "AlertManager configuration reload error\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: PrometheusAlertmanagerConfigNotSynced
+    expr: count(count_values("config_hash", alertmanager_config_hash)) > 1
+    for: 0m
+    labels:
+      severity: warning
+    annotations:
+      summary: Prometheus AlertManager config not synced (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "Configurations of AlertManager cluster instances are out of sync\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: PrometheusAlertmanagerE2eDeadManSwitch
+    expr: vector(1)
+    for: 0m
+    labels:
+      severity: critical
+    annotations:
+      summary: Prometheus AlertManager E2E dead man switch (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "Prometheus DeadManSwitch is an always-firing alert. It's used as an end-to-end test of Prometheus through the Alertmanager.\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: PrometheusNotConnectedToAlertmanager
+    expr: prometheus_notifications_alertmanagers_discovered < 1
+    for: 0m
+    labels:
+      severity: critical
+    annotations:
+      summary: Prometheus not connected to alertmanager (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "Prometheus cannot connect the alertmanager\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: PrometheusAlertmanagerNotificationFailing
+    expr: rate(alertmanager_notifications_failed_total[1m]) > 0
+    for: 0m
+    labels:
+      severity: critical
+    annotations:
+      summary: Prometheus AlertManager notification failing (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "Alertmanager is failing sending notifications\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
diff --git a/roles/monitoring/prometheus/server/templates/prometheus.yml.j2 b/roles/monitoring/prometheus/server/templates/prometheus.yml.j2
index 3975c74d..c76990f4 100644
--- a/roles/monitoring/prometheus/server/templates/prometheus.yml.j2
+++ b/roles/monitoring/prometheus/server/templates/prometheus.yml.j2
@@ -6,6 +6,13 @@ global:
 
 rule_files:
   - /etc/prometheus/rules/*.yml
+{% if prometheus_server_alertmanager is defined %}
+
+alerting:
+  alertmanagers:
+  - static_configs:
+    - targets: ['{{ prometheus_server_alertmanager.url }}']
+{% endif %}
 
 scrape_configs:
   - job_name: 'prometheus'
@@ -13,6 +20,12 @@ scrape_configs:
     - targets: ['localhost:9090']
       labels:
         instance: "{{ inventory_hostname }}"
+{% if prometheus_server_alertmanager is defined %}
+
+  - job_name: 'alertmanager'
+    static_configs:
+    - targets: ['{{ prometheus_server_alertmanager.url }}']
+{% endif %}
 {% for job in prometheus_server_jobs %}
 
   - job_name: '{{ job }}'
-- 
cgit v1.2.3


From 6c990fd148f8813dcbafbf2e27fa5ecbe88af5dc Mon Sep 17 00:00:00 2001
From: Christian Pointner <equinox@spreadspace.org>
Date: Thu, 24 Jun 2021 22:29:26 +0200
Subject: move monitoring web interfaces into common nginx vhost

---
 chaos-at-home/ch-mon.yml                                  | 13 +++++++++++++
 inventory/host_vars/ch-mon.yml                            |  7 +++++++
 roles/monitoring/grafana/tasks/main.yml                   | 15 ---------------
 .../monitoring/prometheus/alertmanager/defaults/main.yml  |  3 +++
 .../templates/prometheus-alertmanager.service.j2          |  2 +-
 roles/monitoring/prometheus/server/defaults/main/main.yml |  4 ++++
 .../prometheus/server/templates/prometheus.service.j2     |  2 +-
 .../prometheus/server/templates/prometheus.yml.j2         |  9 +++++++++
 8 files changed, 38 insertions(+), 17 deletions(-)

(limited to 'inventory/host_vars')

diff --git a/chaos-at-home/ch-mon.yml b/chaos-at-home/ch-mon.yml
index 906e8adc..bb20677f 100644
--- a/chaos-at-home/ch-mon.yml
+++ b/chaos-at-home/ch-mon.yml
@@ -15,3 +15,16 @@
   - role: monitoring/prometheus/alertmanager
   - role: monitoring/prometheus/server
   - role: monitoring/grafana
+  - role: nginx/vhost
+    nginx_vhost:
+      name: monitoring
+      template: generic-proxy-no-buffering
+      hostnames:
+      - "_"
+      locations:
+        '/grafana/':
+          proxy_pass: "http://127.0.0.1:3000"
+        '/prometheus/':
+          proxy_pass: "http://127.0.0.1:9090"
+        '/alertmanager/':
+          proxy_pass: "http://127.0.0.1:9093"
diff --git a/inventory/host_vars/ch-mon.yml b/inventory/host_vars/ch-mon.yml
index 111ffb55..118e7f0b 100644
--- a/inventory/host_vars/ch-mon.yml
+++ b/inventory/host_vars/ch-mon.yml
@@ -63,6 +63,9 @@ prometheus_server_storage:
 
 prometheus_server_alertmanager:
   url: "127.0.0.1:9093"
+  path_prefix: "/alertmanager/"
+
+prometheus_server_web_external_url: /prometheus/
 
 
 prometheus_exporters_extra:
@@ -72,11 +75,15 @@ prometheus_exporter_blackbox_modules_extra:
   icmp:
     prober: icmp
 
+
 promethues_alertmanager_smtp:
   smarthost: "{{ network_zones.lan.prefix | ipaddr(network_zones.lan.offsets['ch-prometheus-legacy']) | ipaddr('address') }}:25"
   from: "noreply@chaos-at-home.org"
   require_tls: no
 
+prometheus_alertmanager_web_route_prefix: /alertmanager/
+
+
 grafana_secret_key: "{{ vault_grafana_secret_key }}"
 
 grafana_config_smtp:
diff --git a/roles/monitoring/grafana/tasks/main.yml b/roles/monitoring/grafana/tasks/main.yml
index 55cce412..8698c036 100644
--- a/roles/monitoring/grafana/tasks/main.yml
+++ b/roles/monitoring/grafana/tasks/main.yml
@@ -79,18 +79,3 @@
     name: grafana-server
     state: started
     enabled: yes
-
-- name: configure nginx vhost
-  vars:
-    nginx_vhost:
-      name: grafana
-      template: generic-proxy-no-buffering
-      hostnames:
-      - "_"
-      locations:
-        '/':
-          proxy_pass: "http://127.0.0.1:{{ grafana_config_server.http_port | default(3000) }}"
-          extra_directives: |-
-            client_max_body_size 0;
-  include_role:
-    name: nginx/vhost
diff --git a/roles/monitoring/prometheus/alertmanager/defaults/main.yml b/roles/monitoring/prometheus/alertmanager/defaults/main.yml
index 34b03df0..62663ab8 100644
--- a/roles/monitoring/prometheus/alertmanager/defaults/main.yml
+++ b/roles/monitoring/prometheus/alertmanager/defaults/main.yml
@@ -3,3 +3,6 @@ promethues_alertmanager_smtp:
   smarthost: "127.0.0.1:25"
   from: "noreply@example.com"
   require_tls: no
+
+prometheus_alertmanager_web_listen_address: 127.0.0.1:9093
+# prometheus_alertmanager_web_route_prefix: /alertmanager/
diff --git a/roles/monitoring/prometheus/alertmanager/templates/prometheus-alertmanager.service.j2 b/roles/monitoring/prometheus/alertmanager/templates/prometheus-alertmanager.service.j2
index f290dca8..e548607d 100644
--- a/roles/monitoring/prometheus/alertmanager/templates/prometheus-alertmanager.service.j2
+++ b/roles/monitoring/prometheus/alertmanager/templates/prometheus-alertmanager.service.j2
@@ -5,7 +5,7 @@ Documentation=https://prometheus.io/docs/alerting/alertmanager/
 [Service]
 Restart=on-failure
 User=prometheus-alertmanager
-ExecStart=/usr/bin/prometheus-alertmanager --config.file=/etc/prometheus/alertmanager.yml --cluster.listen-address= --storage.path="/var/lib/prometheus/alertmanager"
+ExecStart=/usr/bin/prometheus-alertmanager --config.file=/etc/prometheus/alertmanager.yml --cluster.listen-address= --storage.path="/var/lib/prometheus/alertmanager"{% if prometheus_alertmanager_web_route_prefix is defined %} --web.route-prefix={{ prometheus_alertmanager_web_route_prefix }}{% endif %} --web.listen-address={{ prometheus_alertmanager_web_listen_address }}
 ExecReload=/bin/kill -HUP $MAINPID
 TimeoutStopSec=20s
 SendSIGKILL=no
diff --git a/roles/monitoring/prometheus/server/defaults/main/main.yml b/roles/monitoring/prometheus/server/defaults/main/main.yml
index 8e7fea4b..c9291172 100644
--- a/roles/monitoring/prometheus/server/defaults/main/main.yml
+++ b/roles/monitoring/prometheus/server/defaults/main/main.yml
@@ -14,3 +14,7 @@ prometheus_server_rules:
 
 # prometheus_server_alertmanager:
 #   url: "127.0.0.1:9093"
+#   path_prefix: /
+
+prometheus_server_web_listen_address: 127.0.0.1:9090
+# prometheus_server_web_external_url: /prometheus/
diff --git a/roles/monitoring/prometheus/server/templates/prometheus.service.j2 b/roles/monitoring/prometheus/server/templates/prometheus.service.j2
index 0530e589..3a366a61 100644
--- a/roles/monitoring/prometheus/server/templates/prometheus.service.j2
+++ b/roles/monitoring/prometheus/server/templates/prometheus.service.j2
@@ -6,7 +6,7 @@ After=time-sync.target
 [Service]
 Restart=on-failure
 User=prometheus
-ExecStart=/usr/bin/prometheus --config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/var/lib/prometheus/metrics2/ --storage.tsdb.retention.time={{ prometheus_server_retention }}
+ExecStart=/usr/bin/prometheus --config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/var/lib/prometheus/metrics2/ --storage.tsdb.retention.time={{ prometheus_server_retention }}{% if prometheus_server_web_external_url is defined %} --web.external-url={{ prometheus_server_web_external_url }}{% endif %} --web.listen-address={{ prometheus_server_web_listen_address }}
 ExecReload=/bin/kill -HUP $MAINPID
 TimeoutStopSec=20s
 SendSIGKILL=no
diff --git a/roles/monitoring/prometheus/server/templates/prometheus.yml.j2 b/roles/monitoring/prometheus/server/templates/prometheus.yml.j2
index c76990f4..69d5bcdc 100644
--- a/roles/monitoring/prometheus/server/templates/prometheus.yml.j2
+++ b/roles/monitoring/prometheus/server/templates/prometheus.yml.j2
@@ -12,10 +12,16 @@ alerting:
   alertmanagers:
   - static_configs:
     - targets: ['{{ prometheus_server_alertmanager.url }}']
+{%   if 'path_prefix' in prometheus_server_alertmanager %}
+    path_prefix: '{{ prometheus_server_alertmanager.path_prefix }}'
+{%   endif %}
 {% endif %}
 
 scrape_configs:
   - job_name: 'prometheus'
+{% if prometheus_server_web_external_url is defined %}
+    metrics_path: '{{ (prometheus_server_web_external_url | urlsplit('path'), 'metrics') | path_join }}'
+{% endif %}
     static_configs:
     - targets: ['localhost:9090']
       labels:
@@ -23,6 +29,9 @@ scrape_configs:
 {% if prometheus_server_alertmanager is defined %}
 
   - job_name: 'alertmanager'
+{%   if 'path_prefix' in prometheus_server_alertmanager %}
+    metrics_path: '{{ (prometheus_server_alertmanager.path_prefix, 'metrics') | path_join }}'
+{%   endif %}
     static_configs:
     - targets: ['{{ prometheus_server_alertmanager.url }}']
 {% endif %}
-- 
cgit v1.2.3


From 51090aa083e7e7b9c5b3bf78e59cf4d3e9696871 Mon Sep 17 00:00:00 2001
From: Christian Pointner <equinox@spreadspace.org>
Date: Fri, 25 Jun 2021 01:37:52 +0200
Subject: grafana: drop some settings

---
 inventory/host_vars/ch-mon.yml             |  9 ---------
 roles/monitoring/grafana/defaults/main.yml |  3 ---
 roles/monitoring/grafana/tasks/main.yml    | 22 ----------------------
 3 files changed, 34 deletions(-)

(limited to 'inventory/host_vars')

diff --git a/inventory/host_vars/ch-mon.yml b/inventory/host_vars/ch-mon.yml
index 118e7f0b..4df29b23 100644
--- a/inventory/host_vars/ch-mon.yml
+++ b/inventory/host_vars/ch-mon.yml
@@ -85,12 +85,3 @@ prometheus_alertmanager_web_route_prefix: /alertmanager/
 
 
 grafana_secret_key: "{{ vault_grafana_secret_key }}"
-
-grafana_config_smtp:
-  enabled: true
-  host: "{{ network_zones.lan.prefix | ipaddr(network_zones.lan.offsets['ch-prometheus-legacy']) | ipaddr('address') }}:25"
-  from_name: "chaos@home Grafana"
-  from_address: "noreply@chaos-at-home.org"
-
-grafana_config_plugins:
-  enable_alpha: true
diff --git a/roles/monitoring/grafana/defaults/main.yml b/roles/monitoring/grafana/defaults/main.yml
index 0118b8cb..8798dfb5 100644
--- a/roles/monitoring/grafana/defaults/main.yml
+++ b/roles/monitoring/grafana/defaults/main.yml
@@ -19,6 +19,3 @@ grafana_config_security:
 grafana_config_users:
   allow_sign_up: false
   allow_org_create: false
-
-grafana_config_smtp: {}
-grafana_config_plugins: {}
diff --git a/roles/monitoring/grafana/tasks/main.yml b/roles/monitoring/grafana/tasks/main.yml
index 8698c036..61dd8638 100644
--- a/roles/monitoring/grafana/tasks/main.yml
+++ b/roles/monitoring/grafana/tasks/main.yml
@@ -52,28 +52,6 @@
     value: "{{ item.value | string }}"
   notify: restart grafana
 
-- name: configure grafana smtp
-  loop: "{{ grafana_config_smtp | dict2items }}"
-  loop_control:
-    label: "{{ item.key }}"
-  ini_file:
-    path: /etc/grafana/grafana.ini
-    section: smtp
-    option: "{{ item.key }}"
-    value: "{{ item.value | string }}"
-  notify: restart grafana
-
-- name: configure grafana plugins
-  loop: "{{ grafana_config_plugins | dict2items }}"
-  loop_control:
-    label: "{{ item.key }}"
-  ini_file:
-    path: /etc/grafana/grafana.ini
-    section: plugins
-    option: "{{ item.key }}"
-    value: "{{ item.value | string }}"
-  notify: restart grafana
-
 - name: make sure grafan-server is enabled and started
   systemd:
     name: grafana-server
-- 
cgit v1.2.3