Merge branch 'topic/prometheus'

35 files changed, 1396 insertions, 76 deletions

diff --git a/roles/monitoring/grafana/tasks/main.yml b/roles/monitoring/grafana/tasks/main.yml
index 2e7594ec..61dd8638 100644
--- a/roles/monitoring/grafana/tasks/main.yml
+++ b/roles/monitoring/grafana/tasks/main.yml
@@ -46,7 +46,7 @@
   loop_control:
     label: "{{ item.key }}"
   ini_file:
-    path: /etc/grafana/grafana.inig
+    path: /etc/grafana/grafana.ini
     section: users
     option: "{{ item.key }}"
     value: "{{ item.value | string }}"
@@ -57,18 +57,3 @@
     name: grafana-server
     state: started
     enabled: yes
-
-- name: configure nginx vhost
-  vars:
-    nginx_vhost:
-      name: grafana
-      template: generic-proxy-no-buffering
-      hostnames:
-      - "_"
-      locations:
-        '/':
-          proxy_pass: "http://127.0.0.1:{{ grafana_config_server.http_port | default(3000) }}"
-          extra_directives: |-
-            client_max_body_size 0;
-  include_role:
-    name: nginx/vhost
diff --git a/roles/monitoring/prometheus/alertmanager/defaults/main.yml b/roles/monitoring/prometheus/alertmanager/defaults/main.yml
new file mode 100644
index 00000000..62663ab8
--- /dev/null
+++ b/roles/monitoring/prometheus/alertmanager/defaults/main.yml
@@ -0,0 +1,8 @@
+---
+promethues_alertmanager_smtp:
+  smarthost: "127.0.0.1:25"
+  from: "noreply@example.com"
+  require_tls: no
+
+prometheus_alertmanager_web_listen_address: 127.0.0.1:9093
+# prometheus_alertmanager_web_route_prefix: /alertmanager/
diff --git a/roles/monitoring/prometheus/alertmanager/handlers/main.yml b/roles/monitoring/prometheus/alertmanager/handlers/main.yml
new file mode 100644
index 00000000..571b1f7c
--- /dev/null
+++ b/roles/monitoring/prometheus/alertmanager/handlers/main.yml
@@ -0,0 +1,10 @@
+---
+- name: restart prometheus-alertmanager
+  service:
+    name: prometheus-alertmanager
+    state: restarted
+
+- name: reload prometheus-alertmanager
+  service:
+    name: prometheus-alertmanager
+    state: reloaded
diff --git a/roles/monitoring/prometheus/alertmanager/tasks/main.yml b/roles/monitoring/prometheus/alertmanager/tasks/main.yml
new file mode 100644
index 00000000..fe8ce9ca
--- /dev/null
+++ b/roles/monitoring/prometheus/alertmanager/tasks/main.yml
@@ -0,0 +1,45 @@
+---
+- name: check if prometheus apt component of spreadspace repo is enabled
+  assert:
+    msg: "please enable the 'prometheus' component of spreadspace repo using 'spreadspace_apt_repo_components'"
+    that:
+    - spreadspace_apt_repo_components is defined
+    - "'prometheus' in spreadspace_apt_repo_components"
+
+- name: install apt packages
+  apt:
+    name: prom-alertmanager
+    state: present
+
+- name: add user for server
+  user:
+    name: prometheus-alertmanager
+    system: yes
+    home: /nonexistent
+    create_home: no
+
+- name: create data directory
+  file:
+    path: /var/lib/prometheus/alertmanager
+    state: directory
+    owner: prometheus-alertmanager
+    group: prometheus-alertmanager
+
+- name: generate configuration file
+  template:
+    src: alertmanager.yml.j2
+    dest: /etc/prometheus/alertmanager.yml
+  notify: reload prometheus-alertmanager
+
+- name: generate systemd service unit
+  template:
+    src: prometheus-alertmanager.service.j2
+    dest: /etc/systemd/system/prometheus-alertmanager.service
+  notify: restart prometheus-alertmanager
+
+- name: make sure alertmanager is enabled and started
+  systemd:
+    name: prometheus-alertmanager.service
+    daemon_reload: yes
+    state: started
+    enabled: yes
diff --git a/roles/monitoring/prometheus/alertmanager/templates/alertmanager.yml.j2 b/roles/monitoring/prometheus/alertmanager/templates/alertmanager.yml.j2
new file mode 100644
index 00000000..b1d40bb2
--- /dev/null
+++ b/roles/monitoring/prometheus/alertmanager/templates/alertmanager.yml.j2
@@ -0,0 +1,17 @@
+# {{ ansible_managed }}
+
+global:
+  smtp_smarthost: '{{ promethues_alertmanager_smtp.smarthost }}'
+  smtp_from: '{{ promethues_alertmanager_smtp.from }}'
+  smtp_require_tls: {{ promethues_alertmanager_smtp.require_tls | ternary('true', 'false') }}
+
+route:
+  receiver: empty
+
+  routes:
+  - match_re:
+      instance: ^$
+    receiver: empty
+
+receivers:
+- name: empty
diff --git a/roles/monitoring/prometheus/alertmanager/templates/prometheus-alertmanager.service.j2 b/roles/monitoring/prometheus/alertmanager/templates/prometheus-alertmanager.service.j2
new file mode 100644
index 00000000..e548607d
--- /dev/null
+++ b/roles/monitoring/prometheus/alertmanager/templates/prometheus-alertmanager.service.j2
@@ -0,0 +1,37 @@
+[Unit]
+Description=Alertmanager for Prometheus Monitoring system
+Documentation=https://prometheus.io/docs/alerting/alertmanager/
+
+[Service]
+Restart=on-failure
+User=prometheus-alertmanager
+ExecStart=/usr/bin/prometheus-alertmanager --config.file=/etc/prometheus/alertmanager.yml --cluster.listen-address= --storage.path="/var/lib/prometheus/alertmanager"{% if prometheus_alertmanager_web_route_prefix is defined %} --web.route-prefix={{ prometheus_alertmanager_web_route_prefix }}{% endif %} --web.listen-address={{ prometheus_alertmanager_web_listen_address }}
+ExecReload=/bin/kill -HUP $MAINPID
+TimeoutStopSec=20s
+SendSIGKILL=no
+
+# systemd hardening-options
+AmbientCapabilities=
+CapabilityBoundingSet=
+DeviceAllow=/dev/null rw
+DevicePolicy=strict
+LimitMEMLOCK=0
+LimitNOFILE=8192
+LockPersonality=true
+MemoryDenyWriteExecute=true
+NoNewPrivileges=true
+PrivateDevices=true
+PrivateTmp=true
+PrivateUsers=true
+ProtectControlGroups=true
+ProtectHome=true
+ProtectKernelModules=true
+ProtectKernelTunables=true
+ProtectSystem=full
+RemoveIPC=true
+RestrictNamespaces=true
+RestrictRealtime=true
+SystemCallArchitectures=native
+
+[Install]
+WantedBy=multi-user.target
diff --git a/roles/monitoring/prometheus/ca/tasks/main.yml b/roles/monitoring/prometheus/ca/tasks/main.yml
new file mode 100644
index 00000000..064cb6e8
--- /dev/null
+++ b/roles/monitoring/prometheus/ca/tasks/main.yml
@@ -0,0 +1,52 @@
+---
+- name: install python-cryptoraphy
+  apt:
+    name: "{{ python_basename }}-cryptography"
+    state: present
+
+- name: create base directory
+  file:
+    path: /etc/ssl/prometheus
+    state: directory
+
+- name: create CA directory
+  file:
+    path: /etc/ssl/prometheus/ca
+    state: directory
+    owner: root
+    group: root
+    mode: 0700
+
+- name: create CA private key
+  openssl_privatekey:
+    path: /etc/ssl/prometheus/ca/key.pem
+    type: RSA
+    size: 4096
+    owner: root
+    group: root
+    mode: 0600
+
+- name: create signing request for CA certificate
+  openssl_csr:
+    path: /etc/ssl/prometheus/ca/csr.pem
+    privatekey_path: /etc/ssl/prometheus/ca/key.pem
+    CN: "CA for prometheus zone {{ prometheus_zone_name }}"
+    useCommonNameForSAN: no
+    key_usage:
+    - cRLSign
+    - keyCertSign
+    key_usage_critical: yes
+    basic_constraints:
+    - 'CA:TRUE'
+    - 'pathlen:0'
+    basic_constraints_critical: yes
+
+- name: create self-signed CA certificate
+  openssl_certificate:
+    path: /etc/ssl/prometheus/ca-crt.pem
+    csr_path: /etc/ssl/prometheus/ca/csr.pem
+    privatekey_path: /etc/ssl/prometheus/ca/key.pem
+    provider: selfsigned
+    selfsigned_digest: sha256
+    selfsigned_not_after: "+18250d" ## 50 years
+    selfsigned_create_subject_key_identifier: always_create
diff --git a/roles/monitoring/prometheus/exporter/TODO b/roles/monitoring/prometheus/exporter/TODO
new file mode 100644
index 00000000..c02e5699
--- /dev/null
+++ b/roles/monitoring/prometheus/exporter/TODO
@@ -0,0 +1,38 @@
+Node Exporter - Text Collector Scripts:
+ - https://github.com/prometheus-community/node-exporter-textfile-collector-scripts
+ - https://packages.debian.org/bullseye/prometheus-node-exporter-collectors
+
+IPMI Exporter:
+  - https://github.com/soundcloud/ipmi_exporter
+  - https://packages.debian.org/bullseye/prometheus-ipmi-exporter
+
+Postfix Exporter:
+  - https://github.com/kumina/postfix_exporter
+  - https://packages.debian.org/bullseye/prometheus-postfix-exporter
+
+NGINX Exporter:
+  - https://github.com/nginxinc/nginx-prometheus-exporter
+  - https://packages.debian.org/bullseye/prometheus-nginx-exporter
+
+Bind Exporter:
+  - https://github.com/prometheus-community/bind_exporter
+  - https://packages.debian.org/bullseye/prometheus-bind-exporter
+
+MySQLd Exporter:
+  - https://github.com/prometheus/mysqld_exporter
+  - https://packages.debian.org/bullseye/prometheus-mysqld-exporter
+
+Postgres Exporter:
+  - https://github.com/prometheus-community/postgres_exporter
+  - https://packages.debian.org/bullseye/prometheus-postgres-exporter
+
+SNMP Exporter:
+  - https://github.com/prometheus/snmp_exporter
+  - https://packages.debian.org/bullseye/prometheus-snmp-exporter
+
+Process Exporter:
+  - https://github.com/ncabatoff/process-exporter
+  - https://packages.debian.org/bullseye/prometheus-process-exporter
+
+SSL Exporter:
+  - https://github.com/ribbybibby/ssl_exporter
diff --git a/roles/monitoring/prometheus/exporter/base/defaults/main.yml b/roles/monitoring/prometheus/exporter/base/defaults/main.yml
index 5f8ce103..963763a5 100644
--- a/roles/monitoring/prometheus/exporter/base/defaults/main.yml
+++ b/roles/monitoring/prometheus/exporter/base/defaults/main.yml
@@ -1,2 +1,2 @@
 ---
-prometheus_exporter_port: 9000
+prometheus_exporter_listen: ":9999"
diff --git a/roles/monitoring/prometheus/exporter/base/handlers/main.yml b/roles/monitoring/prometheus/exporter/base/handlers/main.yml
new file mode 100644
index 00000000..ebd760cf
--- /dev/null
+++ b/roles/monitoring/prometheus/exporter/base/handlers/main.yml
@@ -0,0 +1,5 @@
+---
+- name: restart prometheus-exporter-exporter
+  service:
+    name: prometheus-exporter-exporter
+    state: restarted
diff --git a/roles/monitoring/prometheus/exporter/base/tasks/main.yml b/roles/monitoring/prometheus/exporter/base/tasks/main.yml
index c3a04bd9..9a214f39 100644
--- a/roles/monitoring/prometheus/exporter/base/tasks/main.yml
+++ b/roles/monitoring/prometheus/exporter/base/tasks/main.yml
@@ -1,16 +1,40 @@
 ---
-- name: create main configuration directories
-  loop:
-  - exporters-available
-  - exporters-enabled
+- name: check if prometheus apt component of spreadspace repo is enabled
+  assert:
+    msg: "please enable the 'prometheus' component of spreadspace repo using 'spreadspace_apt_repo_components'"
+    that:
+    - spreadspace_apt_repo_components is defined
+    - "'prometheus' in spreadspace_apt_repo_components"
+
+- name: install apt packages
+  apt:
+    name: prom-exporter-exporter
+    state: present
+
+- name: create configuration directories
   file:
-    path: "/etc/prometheus-exporter/{{ item }}"
+    path: /etc/prometheus/exporter/enabled
     state: directory
 
-- name: install nginx vhost
-  vars:
-    nginx_vhost:
-      name: prometheus-exporter
-      content: "{{ lookup('template', 'nginx-vhost.conf.j2') }}"
-  include_role:
-    name: nginx/vhost
+- name: add user for prometheus-exporter
+  user:
+    name: prometheus-exporter
+    system: yes
+    home: /nonexistent
+    create_home: no
+
+- name: create TLS certificate and key
+  import_tasks: tls.yml
+
+- name: generate systemd service unit
+  template:
+    src: service.j2
+    dest: /etc/systemd/system/prometheus-exporter-exporter.service
+  notify: restart prometheus-exporter-exporter
+
+- name: make sure prometheus-exporter-exporter is enabled and started
+  systemd:
+    name: prometheus-exporter-exporter.service
+    daemon_reload: yes
+    state: started
+    enabled: yes
diff --git a/roles/monitoring/prometheus/exporter/base/tasks/tls.yml b/roles/monitoring/prometheus/exporter/base/tasks/tls.yml
new file mode 100644
index 00000000..e34025e4
--- /dev/null
+++ b/roles/monitoring/prometheus/exporter/base/tasks/tls.yml
@@ -0,0 +1,100 @@
+---
+- name: install python-cryptoraphy
+  apt:
+    name: "{{ python_basename }}-cryptography"
+    state: present
+
+- name: create base directory
+  file:
+    path: /etc/ssl/prometheus
+    state: directory
+
+- name: create exporter cert/key directory
+  file:
+    path: /etc/ssl/prometheus/exporter
+    state: directory
+    owner: root
+    group: prometheus-exporter
+    mode: 0750
+
+- name: create exporter private key
+  openssl_privatekey:
+    path: /etc/ssl/prometheus/exporter/key.pem
+    type: RSA
+    size: 4096
+    owner: prometheus-exporter
+    group: prometheus-exporter
+    mode: 0400
+  notify: restart prometheus-exporter-exporter
+
+- name: create signing request for exporter certificate
+  openssl_csr:
+    path: /etc/ssl/prometheus/exporter/csr.pem
+    privatekey_path: /etc/ssl/prometheus/exporter/key.pem
+    CN: "{{ inventory_hostname }}"
+    subject_alt_name:
+    - "DNS:{{ host_name }}.{{ host_domain }}"
+    - "IP:{{ ansible_default_ipv4.address }}"
+    key_usage:
+    - digitalSignature
+    key_usage_critical: yes
+    extended_key_usage:
+    - serverAuth
+    extended_key_usage_critical: yes
+    basic_constraints:
+    - 'CA:FALSE'
+    basic_constraints_critical: yes
+
+- name: slurp CSR
+  slurp:
+    src: /etc/ssl/prometheus/exporter/csr.pem
+  register: prometheus_exporter_server_csr
+
+- name: check if exporter certificate exists
+  stat:
+    path: /etc/ssl/prometheus/exporter/crt.pem
+  register: prometheus_exporter_server_cert
+
+- name: read exporter client certificate validity
+  when: prometheus_exporter_server_cert.stat.exists
+  openssl_certificate_info:
+    path: /etc/ssl/prometheus/exporter/crt.pem
+    valid_at:
+      ten_years: '+3650d'
+  register: prometheus_exporter_server_cert_info
+
+- name: slurp existing exporter certificate
+  when: prometheus_exporter_server_cert.stat.exists
+  slurp:
+    src: /etc/ssl/prometheus/exporter/crt.pem
+  register: prometheus_exporter_server_cert_current
+
+- name: generate exporter certificate
+  delegate_to: "{{ prometheus_server }}"
+  community.crypto.x509_certificate_pipe:
+    content: "{{ prometheus_exporter_server_cert_current.content | default('') | b64decode }}"
+    csr_content: "{{ prometheus_exporter_server_csr.content | b64decode }}"
+    provider: ownca
+    ownca_path: /etc/ssl/prometheus/ca-crt.pem
+    ownca_privatekey_path: /etc/ssl/prometheus/ca/key.pem
+    ownca_digest: sha256
+    ownca_not_after: "+18250d" ## 50 years
+    force: "{{ prometheus_exporter_server_cert.stat.exists and (not prometheus_exporter_server_cert_info.valid_at.ten_years) }}"
+  register: prometheus_exporter_server_cert
+
+- name: store exporter certificate
+  copy:
+    content: "{{ prometheus_exporter_server_cert.certificate }}"
+    dest: /etc/ssl/prometheus/exporter/crt.pem
+  notify: restart prometheus-exporter-exporter
+
+- name: slurp CA certificate
+  delegate_to: "{{ prometheus_server }}"
+  slurp:
+    src: /etc/ssl/prometheus/ca-crt.pem
+  register: prometheus_exporter_ca_certificate
+
+- name: install CA certificate
+  copy:
+    content: "{{ prometheus_exporter_ca_certificate.content | b64decode }}"
+    dest: /etc/ssl/prometheus/ca-crt.pem
diff --git a/roles/monitoring/prometheus/exporter/base/templates/nginx-vhost.conf.j2 b/roles/monitoring/prometheus/exporter/base/templates/nginx-vhost.conf.j2
deleted file mode 100644
index e032ca3d..00000000
--- a/roles/monitoring/prometheus/exporter/base/templates/nginx-vhost.conf.j2
+++ /dev/null
@@ -1,15 +0,0 @@
-server {
-  listen {{ prometheus_exporter_port }};
-  listen [::]:{{ prometheus_exporter_port }};
-  server_name _;
-
-  ## TODO: configure ssl
-
-  location / {
-     return 404 "unknown exporter: $uri\n";
-  }
-  include /etc/prometheus-exporter/exporters-enabled/*;
-
-  access_log /var/log/nginx/access-prometheus-exporter.log;
-  error_log /var/log/nginx/error-prometheus-exporter.log;
-}
diff --git a/roles/monitoring/prometheus/exporter/base/templates/service.j2 b/roles/monitoring/prometheus/exporter/base/templates/service.j2
new file mode 100644
index 00000000..c24baf43
--- /dev/null
+++ b/roles/monitoring/prometheus/exporter/base/templates/service.j2
@@ -0,0 +1,32 @@
+[Unit]
+Description=Prometheus exporter proxy
+
+[Service]
+Restart=always
+User=prometheus-exporter
+ExecStart=/usr/bin/prometheus-exporter-exporter -config.dirs=/etc/prometheus/exporter/enabled -config.file="" -web.listen-address="" -web.tls.listen-address="{{ prometheus_exporter_listen }}" -web.tls.cert="/etc/ssl/prometheus/exporter/crt.pem" -web.tls.key="/etc/ssl/prometheus/exporter/key.pem" --web.tls.ca="/etc/ssl/prometheus/ca-crt.pem" -web.tls.verify
+{# TODO: implement reloading once the exporter_exporter supports this #}
+
+# systemd hardening-options
+AmbientCapabilities=
+CapabilityBoundingSet=
+DeviceAllow=/dev/null rw
+DevicePolicy=strict
+LockPersonality=true
+MemoryDenyWriteExecute=true
+NoNewPrivileges=true
+PrivateDevices=true
+PrivateTmp=true
+PrivateUsers=true
+ProtectControlGroups=true
+ProtectHome=true
+ProtectKernelModules=true
+ProtectKernelTunables=true
+ProtectSystem=strict
+RemoveIPC=true
+RestrictNamespaces=true
+RestrictRealtime=true
+SystemCallArchitectures=native
+
+[Install]
+WantedBy=multi-user.target
diff --git a/roles/monitoring/prometheus/exporter/blackbox/defaults/main.yml b/roles/monitoring/prometheus/exporter/blackbox/defaults/main.yml
new file mode 100644
index 00000000..4e7d8d9a
--- /dev/null
+++ b/roles/monitoring/prometheus/exporter/blackbox/defaults/main.yml
@@ -0,0 +1,26 @@
+---
+prometheus_exporter_blackbox_modules:
+  tcp_connect:
+    prober: tcp
+  tcp_tls_connect:
+    prober: tcp
+    tcp:
+      tls: true
+      tls_config:
+        insecure_skip_verify: true
+  http_2xx:
+    prober: http
+  http_tls_2xx:
+    prober: http
+    http:
+      fail_if_not_ssl: true
+      tls_config:
+        insecure_skip_verify: true
+  ssh_banner:
+    prober: tcp
+    tcp:
+      query_response:
+      - expect: "^SSH-2.0-"
+      - send: "SSH-2.0-blackbox-ssh-check"
+
+prometheus_exporter_blackbox_modules_extra: {}
diff --git a/roles/monitoring/prometheus/exporter/blackbox/handlers/main.yml b/roles/monitoring/prometheus/exporter/blackbox/handlers/main.yml
new file mode 100644
index 00000000..99a416e2
--- /dev/null
+++ b/roles/monitoring/prometheus/exporter/blackbox/handlers/main.yml
@@ -0,0 +1,16 @@
+---
+- name: restart prometheus-blackbox-exporter
+  service:
+    name: prometheus-blackbox-exporter
+    state: restarted
+
+- name: reload prometheus-blackbox-exporter
+  service:
+    name: prometheus-blackbox-exporter
+    state: reloaded
+
+- name: reload prometheus-exporter-exporter
+  service:
+    name: prometheus-exporter-exporter
+    ## TODO: implement reload once exporter_exporter supports this...
+    state: restarted
diff --git a/roles/monitoring/prometheus/exporter/blackbox/tasks/main.yml b/roles/monitoring/prometheus/exporter/blackbox/tasks/main.yml
new file mode 100644
index 00000000..3b8e997d
--- /dev/null
+++ b/roles/monitoring/prometheus/exporter/blackbox/tasks/main.yml
@@ -0,0 +1,39 @@
+---
+- name: install apt packages
+  apt:
+    name: prom-exporter-blackbox
+    state: present
+
+- name: create config directory
+  file:
+    path: /etc/prometheus/exporter/blackbox
+    state: directory
+
+- name: generate configuration
+  template:
+    src: config.yml.j2
+    dest: /etc/prometheus/exporter/blackbox/config.yml
+  notify: reload prometheus-blackbox-exporter
+
+- name: generate systemd service unit
+  template:
+    src: service.j2
+    dest: /etc/systemd/system/prometheus-blackbox-exporter.service
+  notify: restart prometheus-blackbox-exporter
+
+- name: make sure prometheus-exporter-exporter is enabled and started
+  systemd:
+    name: prometheus-blackbox-exporter.service
+    daemon_reload: yes
+    state: started
+    enabled: yes
+
+- name: register exporter
+  copy:
+    content: |
+      method: http
+      http:
+        port: 9115
+        path: '/probe'
+    dest: /etc/prometheus/exporter/enabled/blackbox.yml
+  notify: reload prometheus-exporter-exporter
diff --git a/roles/monitoring/prometheus/exporter/blackbox/templates/config.yml.j2 b/roles/monitoring/prometheus/exporter/blackbox/templates/config.yml.j2
new file mode 100644
index 00000000..01e3f7a0
--- /dev/null
+++ b/roles/monitoring/prometheus/exporter/blackbox/templates/config.yml.j2
@@ -0,0 +1,4 @@
+# {{ ansible_managed }}
+
+modules:
+  {{ prometheus_exporter_blackbox_modules | combine(prometheus_exporter_blackbox_modules_extra) | to_nice_yaml(indent=2) | indent(2)}}
diff --git a/roles/monitoring/prometheus/exporter/blackbox/templates/service.j2 b/roles/monitoring/prometheus/exporter/blackbox/templates/service.j2
new file mode 100644
index 00000000..a8a91d0b
--- /dev/null
+++ b/roles/monitoring/prometheus/exporter/blackbox/templates/service.j2
@@ -0,0 +1,36 @@
+[Unit]
+Description=Prometheus blackbox exporter
+
+[Service]
+Restart=always
+User=prometheus-exporter
+ExecStart=/usr/bin/prometheus-blackbox-exporter --web.listen-address="127.0.0.1:9115" --config.file=/etc/prometheus/exporter/blackbox/config.yml
+ExecReload=/bin/kill -HUP $MAINPID
+
+# systemd hardening-options
+{% if prometheus_exporter_blackbox_modules | combine(prometheus_exporter_blackbox_modules_extra) | dict2items | selectattr('value.prober', 'eq', 'icmp') | length > 0 %}
+AmbientCapabilities=CAP_NET_RAW
+CapabilityBoundingSet=CAP_NET_RAW
+{% else %}
+AmbientCapabilities=
+CapabilityBoundingSet=
+{% endif %}
+DeviceAllow=/dev/null rw
+DevicePolicy=strict
+LockPersonality=true
+MemoryDenyWriteExecute=true
+NoNewPrivileges=true
+PrivateDevices=true
+PrivateTmp=true
+ProtectControlGroups=true
+ProtectHome=true
+ProtectKernelModules=true
+ProtectKernelTunables=true
+ProtectSystem=strict
+RemoveIPC=true
+RestrictNamespaces=true
+RestrictRealtime=true
+SystemCallArchitectures=native
+
+[Install]
+WantedBy=multi-user.target
diff --git a/roles/monitoring/prometheus/exporter/meta/main.yml b/roles/monitoring/prometheus/exporter/meta/main.yml
new file mode 100644
index 00000000..d1d3eac7
--- /dev/null
+++ b/roles/monitoring/prometheus/exporter/meta/main.yml
@@ -0,0 +1,7 @@
+---
+dependencies:
+  - role: monitoring/prometheus/exporter/base
+  - role: monitoring/prometheus/exporter/node
+    when: "'node' in (prometheus_exporters_default | union(prometheus_exporters_extra))"
+  - role: monitoring/prometheus/exporter/blackbox
+    when: "'blackbox' in (prometheus_exporters_default | union(prometheus_exporters_extra))"
diff --git a/roles/monitoring/prometheus/exporter/node/defaults/main.yml b/roles/monitoring/prometheus/exporter/node/defaults/main.yml
new file mode 100644
index 00000000..56227fbb
--- /dev/null
+++ b/roles/monitoring/prometheus/exporter/node/defaults/main.yml
@@ -0,0 +1,12 @@
+---
+_prometheus_exporter_node_time_collector_map_:
+  "": timex
+  systemd-timesyncd: timex
+  chrony: ntp
+  openntpd: ntp
+
+prometheus_exporter_node_timesync_collector: "{{ _prometheus_exporter_node_time_collector_map_[ntp_variant | default('')] }}"
+
+prometheus_exporter_node_disable_collectors: []
+prometheus_exporter_node_extra_collectors:
+- "{{ prometheus_exporter_node_timesync_collector }}"
diff --git a/roles/monitoring/prometheus/exporter/node/handlers/main.yml b/roles/monitoring/prometheus/exporter/node/handlers/main.yml
index 9c62baf6..3e1b2000 100644
--- a/roles/monitoring/prometheus/exporter/node/handlers/main.yml
+++ b/roles/monitoring/prometheus/exporter/node/handlers/main.yml
@@ -3,3 +3,9 @@
   service:
     name: prometheus-node-exporter
     state: restarted
+
+- name: reload prometheus-exporter-exporter
+  service:
+    name: prometheus-exporter-exporter
+    ## TODO: implement reload once exporter_exporter supports this...
+    state: restarted
diff --git a/roles/monitoring/prometheus/exporter/node/tasks/main.yml b/roles/monitoring/prometheus/exporter/node/tasks/main.yml
index 286b6d75..c8756acf 100644
--- a/roles/monitoring/prometheus/exporter/node/tasks/main.yml
+++ b/roles/monitoring/prometheus/exporter/node/tasks/main.yml
@@ -1,29 +1,32 @@
 ---
 - name: install apt packages
   apt:
-    name: prometheus-node-exporter
+    name: prom-exporter-node
     state: present
 
-  ## TODO: add other configs
-- name: listen on localhost only
-  lineinfile:
-    path: /etc/default/prometheus-node-exporter
-    regexp: '^ARGS='
-    line: 'ARGS="--web.listen-address=127.0.0.1:9100"'
+- name: create directory for textfile collector
+  file:
+    path: /var/lib/prometheus-node-exporter/textfile-collector
+    state: directory
+
+- name: generate systemd service unit
+  template:
+    src: service.j2
+    dest: /etc/systemd/system/prometheus-node-exporter.service
   notify: restart prometheus-node-exporter
 
-- name: create nginx snippet
+- name: make sure prometheus-exporter-exporter is enabled and started
+  systemd:
+    name: prometheus-node-exporter.service
+    daemon_reload: yes
+    state: started
+    enabled: yes
+
+- name: register exporter
   copy:
     content: |
-      location = /node {
-        proxy_pass http://127.0.0.1:9100/metrics;
-      }
-    dest: /etc/prometheus-exporter/exporters-available/node
-  # notify: reload nginx
-
-- name: enable nginx snippet
-  file:
-    src: /etc/prometheus-exporter/exporters-available/node
-    dest: /etc/prometheus-exporter/exporters-enabled/node
-    state: link
-  # notify: reload nginx
+      method: http
+      http:
+        port: 9100
+    dest: /etc/prometheus/exporter/enabled/node.yml
+  notify: reload prometheus-exporter-exporter
diff --git a/roles/monitoring/prometheus/exporter/node/templates/service.j2 b/roles/monitoring/prometheus/exporter/node/templates/service.j2
new file mode 100644
index 00000000..7aa2834a
--- /dev/null
+++ b/roles/monitoring/prometheus/exporter/node/templates/service.j2
@@ -0,0 +1,10 @@
+[Unit]
+Description=Prometheus node exporter
+
+[Service]
+Restart=always
+User=prometheus-exporter
+ExecStart=/usr/bin/prometheus-node-exporter --web.listen-address="127.0.0.1:9100" --web.disable-exporter-metrics --collector.textfile.directory="/var/lib/prometheus-node-exporter/textfile-collector"{% for collector in prometheus_exporter_node_disable_collectors %} --no-collector.{{ collector }}{% endfor %}{% for collector in prometheus_exporter_node_extra_collectors %} --collector.{{ collector }}{% endfor %}{{ '' }}
+
+[Install]
+WantedBy=multi-user.target
diff --git a/roles/monitoring/prometheus/server/defaults/main.yml b/roles/monitoring/prometheus/server/defaults/main.yml
deleted file mode 100644
index b5d13b5d..00000000
--- a/roles/monitoring/prometheus/server/defaults/main.yml
+++ /dev/null
@@ -1,6 +0,0 @@
----
-# prometheus_server_storage:
-#   type: (zfs|lvm)
-#   ...
-
-prometheus_server_retention: "15d"
diff --git a/roles/monitoring/prometheus/server/defaults/main/main.yml b/roles/monitoring/prometheus/server/defaults/main/main.yml
new file mode 100644
index 00000000..c9291172
--- /dev/null
+++ b/roles/monitoring/prometheus/server/defaults/main/main.yml
@@ -0,0 +1,20 @@
+---
+# prometheus_server_storage:
+#   type: (zfs|lvm)
+#   ...
+
+prometheus_server_retention: "15d"
+
+prometheus_server_jobs:
+  - node
+
+prometheus_server_rules:
+  prometheus: "{{ prometheus_server_rules_prometheus + ((prometheus_server_alertmanager is defined) | ternary(prometheus_server_rules_prometheus_alertmanager, [])) + prometheus_server_rules_prometheus_extra }}"
+  node: "{{ prometheus_server_rules_node + prometheus_server_rules_prometheus_extra }}"
+
+# prometheus_server_alertmanager:
+#   url: "127.0.0.1:9093"
+#   path_prefix: /
+
+prometheus_server_web_listen_address: 127.0.0.1:9090
+# prometheus_server_web_external_url: /prometheus/
diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_node.yml b/roles/monitoring/prometheus/server/defaults/main/rules_node.yml
new file mode 100644
index 00000000..ab7317ac
--- /dev/null
+++ b/roles/monitoring/prometheus/server/defaults/main/rules_node.yml
@@ -0,0 +1,219 @@
+---
+## https://awesome-prometheus-alerts.grep.to/rules#host-and-hardware
+prometheus_server_rules_node_extra: []
+prometheus_server_rules_node:
+  - alert: HostOutOfMemory
+    expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100 < 10
+    for: 2m
+    labels:
+      severity: warning
+    annotations:
+      summary: Host out of memory (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "Node memory is filling up (< 10% left)\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: HostMemoryUnderMemoryPressure
+    expr: rate(node_vmstat_pgmajfault[1m]) > 1000
+    for: 2m
+    labels:
+      severity: warning
+    annotations:
+      summary: Host memory under memory pressure (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "The node is under heavy memory pressure. High rate of major page faults\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: HostOutOfDiskSpace
+    expr: (node_filesystem_avail_bytes * 100) / node_filesystem_size_bytes < 10 and ON (instance, device, mountpoint) node_filesystem_readonly == 0
+    for: 2m
+    labels:
+      severity: warning
+    annotations:
+      summary: Host out of disk space (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "Disk is almost full (< 10% left)\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: HostDiskWillFillIn24Hours
+    expr: (node_filesystem_avail_bytes * 100) / node_filesystem_size_bytes < 10 and ON (instance, device, mountpoint) predict_linear(node_filesystem_avail_bytes{fstype!~"tmpfs"}[1h], 24 * 3600) < 0 and ON (instance, device, mountpoint) node_filesystem_readonly == 0
+    for: 2m
+    labels:
+      severity: warning
+    annotations:
+      summary: Host disk will fill in 24 hours (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "Filesystem is predicted to run out of space within the next 24 hours at current write rate\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: HostOutOfInodes
+    expr: node_filesystem_files_free{mountpoint ="/rootfs"} / node_filesystem_files{mountpoint="/rootfs"} * 100 < 10 and ON (instance, device, mountpoint) node_filesystem_readonly{mountpoint="/rootfs"} == 0
+    for: 2m
+    labels:
+      severity: warning
+    annotations:
+      summary: Host out of inodes (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "Disk is almost running out of available inodes (< 10% left)\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: HostInodesWillFillIn24Hours
+    expr: node_filesystem_files_free{mountpoint ="/rootfs"} / node_filesystem_files{mountpoint="/rootfs"} * 100 < 10 and predict_linear(node_filesystem_files_free{mountpoint="/rootfs"}[1h], 24 * 3600) < 0 and ON (instance, device, mountpoint) node_filesystem_readonly{mountpoint="/rootfs"} == 0
+    for: 2m
+    labels:
+      severity: warning
+    annotations:
+      summary: Host inodes will fill in 24 hours (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "Filesystem is predicted to run out of inodes within the next 24 hours at current write rate\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: HostUnusualDiskReadLatency
+    expr: rate(node_disk_read_time_seconds_total[1m]) / rate(node_disk_reads_completed_total[1m]) > 0.1 and rate(node_disk_reads_completed_total[1m]) > 0
+    for: 2m
+    labels:
+      severity: warning
+    annotations:
+      summary: Host unusual disk read latency (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "Disk latency is growing (read operations > 100ms)\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: HostUnusualDiskWriteLatency
+    expr: rate(node_disk_write_time_seconds_total[1m]) / rate(node_disk_writes_completed_total[1m]) > 0.1 and rate(node_disk_writes_completed_total[1m]) > 0
+    for: 2m
+    labels:
+      severity: warning
+    annotations:
+      summary: Host unusual disk write latency (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "Disk latency is growing (write operations > 100ms)\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: HostHighCpuLoad
+    expr: 100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[2m])) * 100) > 80
+    for: 0m
+    labels:
+      severity: warning
+    annotations:
+      summary: Host high CPU load (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "CPU load is > 80%\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: HostCpuStealNoisyNeighbor
+    expr: avg by(instance) (rate(node_cpu_seconds_total{mode="steal"}[5m])) * 100 > 10
+    for: 0m
+    labels:
+      severity: warning
+    annotations:
+      summary: Host CPU steal noisy neighbor (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "CPU steal is > 10%. A noisy neighbor is killing VM performances or a spot instance may be out of credit.\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: HostSystemdServiceCrashed
+    expr: node_systemd_unit_state{state="failed"} == 1
+    for: 0m
+    labels:
+      severity: warning
+    annotations:
+      summary: Host systemd service crashed (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "systemd service crashed\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: HostPhysicalComponentTooHot
+    expr: node_hwmon_temp_celsius > 75
+    for: 5m
+    labels:
+      severity: warning
+    annotations:
+      summary: Host physical component too hot (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "Physical hardware component too hot\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: HostNodeOvertemperatureAlarm
+    expr: node_hwmon_temp_crit_alarm_celsius == 1
+    for: 0m
+    labels:
+      severity: critical
+    annotations:
+      summary: Host node overtemperature alarm (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "Physical node temperature alarm triggered\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: HostRaidArrayGotInactive
+    expr: node_md_state{state="inactive"} > 0
+    for: 0m
+    labels:
+      severity: critical
+    annotations:
+      summary: Host RAID array got inactive (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "RAID array {{ '{{' }} $labels.device {{ '}}' }} is in degraded state due to one or more disks failures. Number of spare drives is insufficient to fix issue automatically.\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: HostRaidDiskFailure
+    expr: node_md_disks{state="failed"} > 0
+    for: 2m
+    labels:
+      severity: warning
+    annotations:
+      summary: Host RAID disk failure (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "At least one device in RAID array on {{ '{{' }} $labels.instance {{ '}}' }} failed. Array {{ '{{' }} $labels.md_device {{ '}}' }} needs attention and possibly a disk swap\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: HostOomKillDetected
+    expr: increase(node_vmstat_oom_kill[1m]) > 0
+    for: 0m
+    labels:
+      severity: warning
+    annotations:
+      summary: Host OOM kill detected (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "OOM kill detected\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: HostEdacCorrectableErrorsDetected
+    expr: increase(node_edac_correctable_errors_total[1m]) > 0
+    for: 0m
+    labels:
+      severity: info
+    annotations:
+      summary: Host EDAC Correctable Errors detected (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "Host {{ '{{' }} $labels.instance {{ '}}' }} has had {{ '{{' }} printf \"%.0f\" $value {{ '}}' }} correctable memory errors reported by EDAC in the last 5 minutes.\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: HostEdacUncorrectableErrorsDetected
+    expr: node_edac_uncorrectable_errors_total > 0
+    for: 0m
+    labels:
+      severity: warning
+    annotations:
+      summary: Host EDAC Uncorrectable Errors detected (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "Host {{ '{{' }} $labels.instance {{ '}}' }} has had {{ '{{' }} printf \"%.0f\" $value {{ '}}' }} uncorrectable memory errors reported by EDAC in the last 5 minutes.\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: HostNetworkReceiveErrors
+    expr: rate(node_network_receive_errs_total[2m]) / rate(node_network_receive_packets_total[2m]) > 0.01
+    for: 2m
+    labels:
+      severity: warning
+    annotations:
+      summary: Host Network Receive Errors (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "Host {{ '{{' }} $labels.instance {{ '}}' }} interface {{ '{{' }} $labels.device {{ '}}' }} has encountered {{ '{{' }} printf \"%.0f\" $value {{ '}}' }} receive errors in the last five minutes.\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: HostNetworkTransmitErrors
+    expr: rate(node_network_transmit_errs_total[2m]) / rate(node_network_transmit_packets_total[2m]) > 0.01
+    for: 2m
+    labels:
+      severity: warning
+    annotations:
+      summary: Host Network Transmit Errors (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "Host {{ '{{' }} $labels.instance {{ '}}' }} interface {{ '{{' }} $labels.device {{ '}}' }} has encountered {{ '{{' }} printf \"%.0f\" $value {{ '}}' }} transmit errors in the last five minutes.\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: HostNetworkInterfaceSaturated
+    expr: (rate(node_network_receive_bytes_total{device!~"^tap.*"}[1m]) + rate(node_network_transmit_bytes_total{device!~"^tap.*"}[1m])) / node_network_speed_bytes{device!~"^tap.*"} > 0.8
+    for: 1m
+    labels:
+      severity: warning
+    annotations:
+      summary: Host Network Interface Saturated (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "The network interface \"{{ '{{' }} $labels.interface {{ '}}' }}\" on \"{{ '{{' }} $labels.instance {{ '}}' }}\" is getting overloaded.\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: HostConntrackLimit
+    expr: node_nf_conntrack_entries / node_nf_conntrack_entries_limit > 0.8
+    for: 5m
+    labels:
+      severity: warning
+    annotations:
+      summary: Host conntrack limit (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "The number of conntrack is approching limit\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: HostClockSkew
+    expr: (node_timex_offset_seconds > 0.05 and deriv(node_timex_offset_seconds[5m]) >= 0) or (node_timex_offset_seconds < -0.05 and deriv(node_timex_offset_seconds[5m]) <= 0)
+    for: 2m
+    labels:
+      severity: warning
+    annotations:
+      summary: Host clock skew (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "Clock skew detected. Clock is out of sync.\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: HostClockNotSynchronising
+    expr: min_over_time(node_timex_sync_status[1m]) == 0 and node_timex_maxerror_seconds >= 16
+    for: 2m
+    labels:
+      severity: warning
+    annotations:
+      summary: Host clock not synchronising (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "Clock not synchronising.\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_prometheus.yml b/roles/monitoring/prometheus/server/defaults/main/rules_prometheus.yml
new file mode 100644
index 00000000..8d4672b1
--- /dev/null
+++ b/roles/monitoring/prometheus/server/defaults/main/rules_prometheus.yml
@@ -0,0 +1,239 @@
+---
+## https://awesome-prometheus-alerts.grep.to/rules#prometheus-self-monitoring
+prometheus_server_rules_prometheus_extra: []
+prometheus_server_rules_prometheus:
+  - alert: PrometheusJobMissing
+    expr: absent(up{job="prometheus"})
+    for: 0m
+    labels:
+      severity: warning
+    annotations:
+      summary: Prometheus job missing (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "A Prometheus job has disappeared\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: PrometheusJobMissing
+    expr: absent(up{job="prometheus"})
+    for: 0m
+    labels:
+      severity: warning
+    annotations:
+      summary: Prometheus job missing (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "A Prometheus job has disappeared\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: PrometheusTargetMissing
+    expr: up == 0
+    for: 0m
+    labels:
+      severity: critical
+    annotations:
+      summary: Prometheus target missing (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "A Prometheus target has disappeared. An exporter might be crashed.\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: PrometheusAllTargetsMissing
+    expr: count by (job) (up) == 0
+    for: 0m
+    labels:
+      severity: critical
+    annotations:
+      summary: Prometheus all targets missing (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "A Prometheus job does not have living target anymore.\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: PrometheusConfigurationReloadFailure
+    expr: prometheus_config_last_reload_successful != 1
+    for: 0m
+    labels:
+      severity: warning
+    annotations:
+      summary: Prometheus configuration reload failure (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "Prometheus configuration reload error\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: PrometheusTooManyRestarts
+    expr: changes(process_start_time_seconds{job=~"prometheus|pushgateway|alertmanager"}[15m]) > 2
+    for: 0m
+    labels:
+      severity: warning
+    annotations:
+      summary: Prometheus too many restarts (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "Prometheus has restarted more than twice in the last 15 minutes. It might be crashlooping.\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: PrometheusRuleEvaluationFailures
+    expr: increase(prometheus_rule_evaluation_failures_total[3m]) > 0
+    for: 0m
+    labels:
+      severity: critical
+    annotations:
+      summary: Prometheus rule evaluation failures (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "Prometheus encountered {{ '{{' }} $value {{ '}}' }} rule evaluation failures, leading to potentially ignored alerts.\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: PrometheusTemplateTextExpansionFailures
+    expr: increase(prometheus_template_text_expansion_failures_total[3m]) > 0
+    for: 0m
+    labels:
+      severity: critical
+    annotations:
+      summary: Prometheus template text expansion failures (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "Prometheus encountered {{ '{{' }} $value {{ '}}' }} template text expansion failures\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: PrometheusRuleEvaluationSlow
+    expr: prometheus_rule_group_last_duration_seconds > prometheus_rule_group_interval_seconds
+    for: 5m
+    labels:
+      severity: warning
+    annotations:
+      summary: Prometheus rule evaluation slow (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "Prometheus rule evaluation took more time than the scheduled interval. It indicates a slower storage backend access or too complex query.\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: PrometheusNotificationsBacklog
+    expr: min_over_time(prometheus_notifications_queue_length[10m]) > 0
+    for: 0m
+    labels:
+      severity: warning
+    annotations:
+      summary: Prometheus notifications backlog (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "The Prometheus notification queue has not been empty for 10 minutes\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: PrometheusTargetEmpty
+    expr: prometheus_sd_discovered_targets == 0
+    for: 0m
+    labels:
+      severity: critical
+    annotations:
+      summary: Prometheus target empty (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "Prometheus has no target in service discovery\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: PrometheusTargetScrapingSlow
+    expr: prometheus_target_interval_length_seconds{quantile="0.9"} > 60
+    for: 5m
+    labels:
+      severity: warning
+    annotations:
+      summary: Prometheus target scraping slow (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "Prometheus is scraping exporters slowly\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: PrometheusLargeScrape
+    expr: increase(prometheus_target_scrapes_exceeded_sample_limit_total[10m]) > 10
+    for: 5m
+    labels:
+      severity: warning
+    annotations:
+      summary: Prometheus large scrape (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "Prometheus has many scrapes that exceed the sample limit\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: PrometheusTargetScrapeDuplicate
+    expr: increase(prometheus_target_scrapes_sample_duplicate_timestamp_total[5m]) > 0
+    for: 0m
+    labels:
+      severity: warning
+    annotations:
+      summary: Prometheus target scrape duplicate (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "Prometheus has many samples rejected due to duplicate timestamps but different values\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: PrometheusTsdbCheckpointCreationFailures
+    expr: increase(prometheus_tsdb_checkpoint_creations_failed_total[1m]) > 0
+    for: 0m
+    labels:
+      severity: critical
+    annotations:
+      summary: Prometheus TSDB checkpoint creation failures (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "Prometheus encountered {{ '{{' }} $value {{ '}}' }} checkpoint creation failures\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: PrometheusTsdbCheckpointDeletionFailures
+    expr: increase(prometheus_tsdb_checkpoint_deletions_failed_total[1m]) > 0
+    for: 0m
+    labels:
+      severity: critical
+    annotations:
+      summary: Prometheus TSDB checkpoint deletion failures (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "Prometheus encountered {{ '{{' }} $value {{ '}}' }} checkpoint deletion failures\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: PrometheusTsdbCompactionsFailed
+    expr: increase(prometheus_tsdb_compactions_failed_total[1m]) > 0
+    for: 0m
+    labels:
+      severity: critical
+    annotations:
+      summary: Prometheus TSDB compactions failed (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "Prometheus encountered {{ '{{' }} $value {{ '}}' }} TSDB compactions failures\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: PrometheusTsdbHeadTruncationsFailed
+    expr: increase(prometheus_tsdb_head_truncations_failed_total[1m]) > 0
+    for: 0m
+    labels:
+      severity: critical
+    annotations:
+      summary: Prometheus TSDB head truncations failed (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "Prometheus encountered {{ '{{' }} $value {{ '}}' }} TSDB head truncation failures\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: PrometheusTsdbReloadFailures
+    expr: increase(prometheus_tsdb_reloads_failures_total[1m]) > 0
+    for: 0m
+    labels:
+      severity: critical
+    annotations:
+      summary: Prometheus TSDB reload failures (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "Prometheus encountered {{ '{{' }} $value {{ '}}' }} TSDB reload failures\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: PrometheusTsdbWalCorruptions
+    expr: increase(prometheus_tsdb_wal_corruptions_total[1m]) > 0
+    for: 0m
+    labels:
+      severity: critical
+    annotations:
+      summary: Prometheus TSDB WAL corruptions (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "Prometheus encountered {{ '{{' }} $value {{ '}}' }} TSDB WAL corruptions\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: PrometheusTsdbWalTruncationsFailed
+    expr: increase(prometheus_tsdb_wal_truncations_failed_total[1m]) > 0
+    for: 0m
+    labels:
+      severity: critical
+    annotations:
+      summary: Prometheus TSDB WAL truncations failed (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "Prometheus encountered {{ '{{' }} $value {{ '}}' }} TSDB WAL truncation failures\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+
+prometheus_server_rules_prometheus_alertmanager:
+  - alert: PrometheusAlertmanagerConfigurationReloadFailure
+    expr: alertmanager_config_last_reload_successful != 1
+    for: 0m
+    labels:
+      severity: warning
+    annotations:
+      summary: Prometheus AlertManager configuration reload failure (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "AlertManager configuration reload error\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: PrometheusAlertmanagerConfigNotSynced
+    expr: count(count_values("config_hash", alertmanager_config_hash)) > 1
+    for: 0m
+    labels:
+      severity: warning
+    annotations:
+      summary: Prometheus AlertManager config not synced (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "Configurations of AlertManager cluster instances are out of sync\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: PrometheusAlertmanagerE2eDeadManSwitch
+    expr: vector(1)
+    for: 0m
+    labels:
+      severity: critical
+    annotations:
+      summary: Prometheus AlertManager E2E dead man switch (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "Prometheus DeadManSwitch is an always-firing alert. It's used as an end-to-end test of Prometheus through the Alertmanager.\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: PrometheusNotConnectedToAlertmanager
+    expr: prometheus_notifications_alertmanagers_discovered < 1
+    for: 0m
+    labels:
+      severity: critical
+    annotations:
+      summary: Prometheus not connected to alertmanager (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "Prometheus cannot connect the alertmanager\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
+
+  - alert: PrometheusAlertmanagerNotificationFailing
+    expr: rate(alertmanager_notifications_failed_total[1m]) > 0
+    for: 0m
+    labels:
+      severity: critical
+    annotations:
+      summary: Prometheus AlertManager notification failing (instance {{ '{{' }} $labels.instance {{ '}}' }})
+      description: "Alertmanager is failing sending notifications\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"
diff --git a/roles/monitoring/prometheus/server/filter_plugins/prometheus.py b/roles/monitoring/prometheus/server/filter_plugins/prometheus.py
new file mode 100644
index 00000000..81cfae70
--- /dev/null
+++ b/roles/monitoring/prometheus/server/filter_plugins/prometheus.py
@@ -0,0 +1,29 @@
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+from functools import partial
+
+from ansible import errors
+
+
+def prometheus_job_targets(hostvars, jobs, targets):
+    try:
+        result = []
+        for job in jobs:
+            for target in targets:
+                enabled = job in hostvars[target]['prometheus_exporters_default'] or job in hostvars[target]['prometheus_exporters_extra']
+                result.append({'job': job, 'target': target, 'enabled': enabled})
+        return result
+    except Exception as e:
+        raise errors.AnsibleFilterError("prometheus_job_targets(): %s" % str(e))
+
+
+class FilterModule(object):
+
+    ''' prometheus filters '''
+    filter_map = {
+        'prometheus_job_targets': prometheus_job_targets,
+    }
+
+    def filters(self):
+        return self.filter_map
diff --git a/roles/monitoring/prometheus/server/handlers/main.yml b/roles/monitoring/prometheus/server/handlers/main.yml
index edeba752..bf8735e9 100644
--- a/roles/monitoring/prometheus/server/handlers/main.yml
+++ b/roles/monitoring/prometheus/server/handlers/main.yml
@@ -3,3 +3,8 @@
   service:
     name: prometheus
     state: restarted
+
+- name: reload prometheus
+  service:
+    name: prometheus
+    state: reloaded
diff --git a/roles/monitoring/prometheus/server/tasks/main.yml b/roles/monitoring/prometheus/server/tasks/main.yml
index 784e872a..a70bd6fd 100644
--- a/roles/monitoring/prometheus/server/tasks/main.yml
+++ b/roles/monitoring/prometheus/server/tasks/main.yml
@@ -1,4 +1,11 @@
 ---
+- name: check if prometheus apt component of spreadspace repo is enabled
+  assert:
+    msg: "please enable the 'prometheus' component of spreadspace repo using 'spreadspace_apt_repo_components'"
+    that:
+    - spreadspace_apt_repo_components is defined
+    - "'prometheus' in spreadspace_apt_repo_components"
+
 - name: prepare storage volume for /var/lib/prometheus
   when: prometheus_server_storage is defined
   vars:
@@ -8,12 +15,83 @@
 
 - name: install apt packages
   apt:
-    name: prometheus
+    name: prom-server
     state: present
 
-- name: listen on localhost only
-  lineinfile:
-    path: /etc/default/prometheus
-    regexp: '^ARGS='
-    line: 'ARGS="--web.listen-address=127.0.0.1:9090 --storage.tsdb.retention={{ prometheus_server_retention }}"'
+- name: add user for server
+  user:
+    name: prometheus
+    system: yes
+    home: /var/lib/prometheus
+    create_home: no
+
+- name: create data directory
+  file:
+    path: /var/lib/prometheus/metrics2
+    state: directory
+    owner: prometheus
+    group: prometheus
+
+- name: create TLS CA and certificates
+  import_tasks: tls.yml
+
+- name: create configuration directories
+  loop:
+  - jobs
+  - rules
+  - targets
+  file:
+    path: "/etc/prometheus/{{ item }}"
+    state: directory
+
+- name: create sub-directroy for all exporter types in jobs directory
+  loop: "{{ prometheus_server_jobs }}"
+  file:
+    path: "/etc/prometheus/jobs/{{ item }}"
+    state: directory
+
+- name: generate targets config
+  loop: "{{ prometheus_zone_targets }}"
+  copy:
+    content: |
+      - targets: [ "{{ hostvars[item].prometheus_scrape_endpoint }}" ]
+        labels:
+          instance: "{{ item }}"
+    dest: "/etc/prometheus/targets/{{ item }}.yml"
+
+- name: enable targets for jobs
+  loop: "{{ hostvars | prometheus_job_targets(prometheus_server_jobs, prometheus_zone_targets) }}"
+  loop_control:
+    label: "{{ item.job }} -> {{ item.target }}"
+  file:
+    src: "{{ item.enabled | ternary('/etc/prometheus/targets/' + item.target + '.yml', omit) }}"
+    path: "/etc/prometheus/jobs/{{ item.job }}/{{ item.target }}.yml"
+    state: "{{ item.enabled | ternary('link', 'absent') }}"
+
+- name: generate rules files for all jobs
+  loop: "{{ prometheus_server_jobs | union(['prometheus']) }}"
+  template:
+    src: rules.yml.j2
+    dest: "/etc/prometheus/rules/{{ item }}.yml"
+    validate: "promtool check rules %s"
+  notify: reload prometheus
+
+- name: generate configuration file
+  template:
+    src: prometheus.yml.j2
+    dest: /etc/prometheus/prometheus.yml
+    validate: "promtool check config %s"
+  notify: reload prometheus
+
+- name: generate systemd service unit
+  template:
+    src: prometheus.service.j2
+    dest: /etc/systemd/system/prometheus.service
   notify: restart prometheus
+
+- name: make sure prometheus is enabled and started
+  systemd:
+    name: prometheus.service
+    daemon_reload: yes
+    state: started
+    enabled: yes
diff --git a/roles/monitoring/prometheus/server/tasks/tls.yml b/roles/monitoring/prometheus/server/tasks/tls.yml
new file mode 100644
index 00000000..940c69b1
--- /dev/null
+++ b/roles/monitoring/prometheus/server/tasks/tls.yml
@@ -0,0 +1,75 @@
+---
+- name: install python-cryptoraphy
+  apt:
+    name: "{{ python_basename }}-cryptography"
+    state: present
+
+- name: create base directory
+  file:
+    path: /etc/ssl/prometheus
+    state: directory
+
+- name: create server cert/key directory
+  file:
+    path: /etc/ssl/prometheus/server
+    state: directory
+    owner: root
+    group: prometheus
+    mode: 0750
+
+- name: create private key for scrape-client certificate
+  openssl_privatekey:
+    path: /etc/ssl/prometheus/server/scrape-key.pem
+    type: RSA
+    size: 4096
+    owner: prometheus
+    group: prometheus
+    mode: 0400
+  notify: reload prometheus
+
+- name: create signing request for scrape-client certificate
+  openssl_csr:
+    path: /etc/ssl/prometheus/server/scrape-csr.pem
+    privatekey_path: /etc/ssl/prometheus/server/scrape-key.pem
+    CN: "{{ inventory_hostname }}"
+    subject_alt_name:
+    - "DNS:{{ host_name }}.{{ host_domain }}"
+    - "IP:{{ ansible_default_ipv4.address }}"
+    key_usage:
+    - digitalSignature
+    key_usage_critical: yes
+    extended_key_usage:
+    - clientAuth
+    extended_key_usage_critical: yes
+    basic_constraints:
+    - 'CA:FALSE'
+    basic_constraints_critical: yes
+
+## TODO: install /etc/ssl/prometheus/ca-crt.pem from CA host
+
+- name: check if scrape-client certificate exists
+  stat:
+    path: /etc/ssl/prometheus/server/scrape-crt.pem
+  register: prometheus_server_scrape_client_cert
+
+- name: check scrape-client certificate validity
+  when: prometheus_server_scrape_client_cert.stat.exists
+  openssl_certificate_info:
+    path: /etc/ssl/prometheus/server/scrape-crt.pem
+    valid_at:
+      ten_years: '+3650d'
+  register: prometheus_server_scrape_client_cert_info
+
+## TODO: implement remote signing?
+
+- name: create scrape-client certificate
+  openssl_certificate:
+    path: /etc/ssl/prometheus/server/scrape-crt.pem
+    csr_path: /etc/ssl/prometheus/server/scrape-csr.pem
+    provider: ownca
+    ownca_path: /etc/ssl/prometheus/ca-crt.pem
+    ownca_privatekey_path: /etc/ssl/prometheus/ca/key.pem
+    ownca_digest: sha256
+    ownca_not_after: "+18250d" ## 50 years
+    force: "{{ prometheus_server_scrape_client_cert.stat.exists and (not prometheus_server_scrape_client_cert_info.valid_at.ten_years) }}"
+  notify: reload prometheus
diff --git a/roles/monitoring/prometheus/server/templates/prometheus.service.j2 b/roles/monitoring/prometheus/server/templates/prometheus.service.j2
new file mode 100644
index 00000000..3a366a61
--- /dev/null
+++ b/roles/monitoring/prometheus/server/templates/prometheus.service.j2
@@ -0,0 +1,38 @@
+[Unit]
+Description=Monitoring system and time series database
+Documentation=https://prometheus.io/docs/introduction/overview/ man:prometheus(1)
+After=time-sync.target
+
+[Service]
+Restart=on-failure
+User=prometheus
+ExecStart=/usr/bin/prometheus --config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/var/lib/prometheus/metrics2/ --storage.tsdb.retention.time={{ prometheus_server_retention }}{% if prometheus_server_web_external_url is defined %} --web.external-url={{ prometheus_server_web_external_url }}{% endif %} --web.listen-address={{ prometheus_server_web_listen_address }}
+ExecReload=/bin/kill -HUP $MAINPID
+TimeoutStopSec=20s
+SendSIGKILL=no
+
+# systemd hardening-options
+AmbientCapabilities=
+CapabilityBoundingSet=
+DeviceAllow=/dev/null rw
+DevicePolicy=strict
+LimitMEMLOCK=0
+LimitNOFILE=8192
+LockPersonality=true
+MemoryDenyWriteExecute=true
+NoNewPrivileges=true
+PrivateDevices=true
+PrivateTmp=true
+PrivateUsers=true
+ProtectControlGroups=true
+ProtectHome=true
+ProtectKernelModules=true
+ProtectKernelTunables=true
+ProtectSystem=full
+RemoveIPC=true
+RestrictNamespaces=true
+RestrictRealtime=true
+SystemCallArchitectures=native
+
+[Install]
+WantedBy=multi-user.target
diff --git a/roles/monitoring/prometheus/server/templates/prometheus.yml.j2 b/roles/monitoring/prometheus/server/templates/prometheus.yml.j2
new file mode 100644
index 00000000..69d5bcdc
--- /dev/null
+++ b/roles/monitoring/prometheus/server/templates/prometheus.yml.j2
@@ -0,0 +1,121 @@
+# {{ ansible_managed }}
+
+global:
+  scrape_interval:     15s
+  evaluation_interval: 15s
+
+rule_files:
+  - /etc/prometheus/rules/*.yml
+{% if prometheus_server_alertmanager is defined %}
+
+alerting:
+  alertmanagers:
+  - static_configs:
+    - targets: ['{{ prometheus_server_alertmanager.url }}']
+{%   if 'path_prefix' in prometheus_server_alertmanager %}
+    path_prefix: '{{ prometheus_server_alertmanager.path_prefix }}'
+{%   endif %}
+{% endif %}
+
+scrape_configs:
+  - job_name: 'prometheus'
+{% if prometheus_server_web_external_url is defined %}
+    metrics_path: '{{ (prometheus_server_web_external_url | urlsplit('path'), 'metrics') | path_join }}'
+{% endif %}
+    static_configs:
+    - targets: ['localhost:9090']
+      labels:
+        instance: "{{ inventory_hostname }}"
+{% if prometheus_server_alertmanager is defined %}
+
+  - job_name: 'alertmanager'
+{%   if 'path_prefix' in prometheus_server_alertmanager %}
+    metrics_path: '{{ (prometheus_server_alertmanager.path_prefix, 'metrics') | path_join }}'
+{%   endif %}
+    static_configs:
+    - targets: ['{{ prometheus_server_alertmanager.url }}']
+{% endif %}
+{% for job in prometheus_server_jobs %}
+
+  - job_name: '{{ job }}'
+    metrics_path: /proxy
+    params:
+      module:
+      - {{ job }}
+    scheme: https
+    tls_config:
+      ca_file: /etc/ssl/prometheus/ca-crt.pem
+      cert_file: /etc/ssl/prometheus/server/scrape-crt.pem
+      key_file: /etc/ssl/prometheus/server/scrape-key.pem
+    file_sd_configs:
+    - files:
+      - "/etc/prometheus/jobs/{{ job }}/*.yml"
+{% endfor %}
+
+  ## TODO: temporary test
+  - job_name: 'ping'
+    metrics_path: /proxy
+    params:
+      module:
+      - blackbox
+      - icmp
+    scheme: https
+    tls_config:
+      ca_file: /etc/ssl/prometheus/ca-crt.pem
+      cert_file: /etc/ssl/prometheus/server/scrape-crt.pem
+      key_file: /etc/ssl/prometheus/server/scrape-key.pem
+    static_configs:
+      - targets:
+        - 62.99.185.129
+        - 9.9.9.9
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: 192.168.32.230:9999
+
+  - job_name: 'https'
+    metrics_path: /proxy
+    params:
+      module:
+      - blackbox
+      - http_tls_2xx
+    scheme: https
+    tls_config:
+      ca_file: /etc/ssl/prometheus/ca-crt.pem
+      cert_file: /etc/ssl/prometheus/server/scrape-crt.pem
+      key_file: /etc/ssl/prometheus/server/scrape-key.pem
+    static_configs:
+      - targets:
+        - web.chaos-at-home.org
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: 192.168.32.230:9999
+
+  - job_name: 'ssh'
+    metrics_path: /proxy
+    params:
+      module:
+      - blackbox
+      - ssh_banner
+    scheme: https
+    tls_config:
+      ca_file: /etc/ssl/prometheus/ca-crt.pem
+      cert_file: /etc/ssl/prometheus/server/scrape-crt.pem
+      key_file: /etc/ssl/prometheus/server/scrape-key.pem
+    static_configs:
+      - targets:
+        - 192.168.32.230:222
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - target_label: instance
+        replacement: 'ch-mon'
+      - target_label: __address__
+        replacement: 192.168.32.230:9999
diff --git a/roles/monitoring/prometheus/server/templates/rules.yml.j2 b/roles/monitoring/prometheus/server/templates/rules.yml.j2
new file mode 100644
index 00000000..30576363
--- /dev/null
+++ b/roles/monitoring/prometheus/server/templates/rules.yml.j2
@@ -0,0 +1,5 @@
+# {{ ansible_managed }}
+
+groups:
+  - name: {{ item }}
+    {{ {'rules': prometheus_server_rules[item]} | to_nice_yaml(indent=2, width=1337) | indent(4) }}