path: root/inventory/host_vars/ch-mon.yml

---
install_jumphost: ch-jump

system_lvm_volume_size_root: 3G
install:
  vm:
    memory: 8G
    numcpus: 8
    autostart: yes
  disks:
    primary: /dev/sda
    scsi:
      sda:
        type: zfs
        name: root
        size: 10g
      sdb:
        type: zfs
        name: data
        size: 50g
  interfaces:
  - bridge: br-svc
    name: svc0
  - bridge: br-iot
    name: iot0
  - bridge: br-mgmt
    name: mgmt0

network:
  nameservers: "{{ network_zones.svc.dns }}"
  domain: "{{ host_domain }}"
  systemd_link:
    interfaces: "{{ install.interfaces }}"
  primary: &_network_primary_
    name: svc0
    address: "{{ network_zones.svc.prefix | ansible.utils.ipaddr(network_zones.svc.offsets[inventory_hostname]) }}"
    gateway: "{{ network_zones.svc.gateway }}"
    static_routes:
    - destination: "{{ network_zones.lan.prefix }}"
      gateway: "{{ network_zones.svc.prefix | ansible.utils.ipaddr(network_zones.svc.offsets['ch-gw-lan']) | ansible.utils.ipaddr('address') }}"
  interfaces:
  - *_network_primary_
  - name: iot0
    address: "{{ network_zones.iot.prefix | ansible.utils.ipaddr(network_zones.iot.offsets[inventory_hostname]) }}"
  - name: mgmt0
    address: "{{ network_zones.mgmt.prefix | ansible.utils.ipaddr(network_zones.mgmt.offsets[inventory_hostname]) }}"


lvm_groups:
  mondata:
    pvs:
    - /dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_drive-scsi0-0-0-1


spreadspace_apt_repo_components:
  - main
  - prometheus


nftables_base_rules:
  main: |
    table inet global {
      chain input_iot {
        ip saddr != {{ network_zones.iot.prefix }} drop
        ip protocol icmp accept
        ip6 nexthdr ipv6-icmp accept
      }

      chain input_mgmt {
        ip saddr != {{ network_zones.mgmt.prefix }} drop
        ip protocol icmp accept
        ip6 nexthdr ipv6-icmp accept
      }

      chain input {
        type filter hook input priority filter; policy drop;
        ct state vmap { established: accept, related: accept, invalid: drop }
        iifname vmap { lo: accept, svc0: accept, iot0: jump input_iot, mgmt0: jump input_mgmt }
      }

      chain forward {
        type filter hook forward priority 0; policy drop;
      }
    }
  protect-grafana-auth-proxy: |
    table inet filter {
      chain protect-grafana-auth-proxy {
        type filter hook output priority filter; policy accept;
        meta skuid != { root, www-data } tcp dport 3000 counter reject
      }
    }

whawty_nginx_sso_backends:
  chaos-at-home:
    port: 1234
    login_url: https://login.chaos-at-home.org/login

whawty_nginx_sso_auths:
  chaos-at-home:
    config:
      cookie:
        name: __Secure-chaos-at-home-sso
        keys:
        - name: 2023-11
          ed25519:
            public-key-data: |-
              -----BEGIN PUBLIC KEY-----
              MCowBQYDK2VwAyEAawvVwThGnYYBDLjQ0Rs71prAmxQ/tfaPUNZvPWS3Z3U=
              -----END PUBLIC KEY-----
        backend:
          bolt: {}
          sync:
            base-url: "https://{{ network_services.http.addr }}"
            http-host: "login.chaos-at-home.org"
            token: "{{  vault_whawty_nginx_sso_sync_tokens['chaos-at-home'][inventory_hostname] }}"
            tls:
              server-name: "login.chaos-at-home.org"
      web:
        listen: 127.0.0.1:1234
      prometheus: {}

prometheus_job_multitarget_whawty_nginx_sso:
  ch-mon:
  - app_instance: chaos-at-home


prometheus_server_storage:
  type: lvm
  vg: mondata
  lv: prometheus
  size: 30G
  fs: ext4

prometheus_server_external_labels:
  environment: chaos-at-home
  monitor: "{{ inventory_hostname }}"

prometheus_server_alertmanager:
  url: "127.0.0.1:9093"
  path_prefix: "/alertmanager/"
  basic_auth:
    username: server
    password: "{{ vault_prometheus_alertmanager_auth_user_passwords['server'] }}"

prometheus_server_web_external_url: "http://mon.chaos-at-home.org/prometheus/"

prometheus_server_auth_users:
  server: "{{ vault_prometheus_server_auth_user_passwords['server'] }}"
  grafana: "{{ vault_prometheus_server_auth_user_passwords['grafana'] }}"
  proxy: "{{ vault_prometheus_server_auth_user_passwords['proxy'] }}"

prometheus_server_selfscraping_auth:
  username: server
  password: "{{ vault_prometheus_server_auth_user_passwords['server'] }}"


prometheus_exporters_extra:
  - blackbox
  - nut
  - ssl
  - smokeping


prometheus_exporter_smokeping_targets:
  - hosts:
    - "{{ network_zones.magenta.gateway }}"
    - "{{ network_zones.magenta.dns[0] }}"
    - "{{ network_zones.magenta.dns[1] }}"
    - 9.9.9.9
    - 8.8.8.8
    - 1.1.1.1
    - "{{ hostvars['ch-atlas'].vm_host.network.bridges.public.prefix | ansible.utils.ipaddr(hostvars['ch-atlas'].vm_host.network.bridges.public.offsets['ch-atlas']) | ansible.utils.ipaddr('address') }}"

prometheus_job_multitarget_blackbox__probe:
  ch-mon:
  - svc_kind: ssh
    svc_instance: "{{ inventory_hostname }}"
    target: "{{ network_zones.svc.prefix | ansible.utils.ipaddr(network_zones.svc.offsets[inventory_hostname]) | ansible.utils.ipaddr('address') }}:{{ ansible_port | default(22) }}"
    module: ssh_banner
  - svc_kind: https
    svc_instance: "mon.chaos-at-home.org"
    target: "https://{{ network_zones.svc.prefix | ansible.utils.ipaddr(network_zones.svc.offsets[inventory_hostname]) | ansible.utils.ipaddr('address') }}/healthz"
    module: http_tls_2xx

prometheus_job_multitarget_ssl__probe:
  ch-mon:
  - module: file
    target: "/etc/ssl/prometheus/**/*.pem"
    sslcert_instance: prometheus


prometheus_server_rules_node_extra:
  - alert: GitFsckMetricsOutdated
    expr: time() - git_fsck_run > 100000
    for: 0m
    labels:
      severity: warning
    annotations:
      summary: Metrics from git-fsck are too old (instance {{ '{{' }} $labels.instance {{ '}}' }})
      description: "The exported values from git-fsck on host {{ '{{' }} $labels.instance {{ '}}' }} are {{ '{{' }} $value {{ '}}' }} seconds old.\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"

  - alert: GitFsckFailed
    expr: git_fsck_failed != 0
    for: 0m
    labels:
      severity: warning
    annotations:
      summary: git-filesystem check failed (instance {{ '{{' }} $labels.instance {{ '}}' }})
      description: "The git repostory {{ '{{' }} $labels.repository {{ '}}' }}@{{ '{{' }} $labels.gitolite_instance {{ '}}' }} on host {{ '{{' }} $labels.instance {{ '}}' }} is corrupt.\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS = {{ '{{' }} $labels {{ '}}' }}"


prometheus_alertmanager_smtp:
  smarthost: "{{ network_zones.lan.prefix | ansible.utils.ipaddr(network_zones.lan.offsets['ch-prometheus-legacy']) | ansible.utils.ipaddr('address') }}:25"
  from: "noreply@chaos-at-home.org"
  require_tls: no

prometheus_alertmanager_web_external_url: "http://mon.chaos-at-home.org/alertmanager/"

prometheus_alertmanager_auth_users:
  server: "{{ vault_prometheus_alertmanager_auth_user_passwords['server'] }}"
  proxy: "{{ vault_prometheus_alertmanager_auth_user_passwords['proxy'] }}"

prometheus_alertmanager_route:
  receiver: empty
  routes:
  - receiver: equinox-mail
    matchers:
    - 'alertname != PrometheusAlertmanagerE2eDeadManSwitch'
    group_by:
    - instance

prometheus_alertmanager_receivers:
  - name: empty
  - name: equinox-mail
    email_configs:
    - to: equinox@chaos-at-home.org
      send_resolved: yes


grafana_secret_key: "{{ vault_grafana_secret_key }}"

grafana_config_auth:
  disable_signout_menu: true

grafana_config_auth_proxy:
  enabled: true
  whitelist: 127.0.0.1

grafana_datasources:
 - name: "Prometheus"
   type: "prometheus"
   access: "proxy"
   url: "http://127.0.0.1:9090/prometheus"
   basicAuth: true
   basicAuthUser: "grafana"
   isDefault: yes
   secureJsonData:
     basicAuthPassword: "{{ vault_prometheus_server_auth_user_passwords['grafana'] }}"
   jsonData:
     manageAlerts: no

grafana_dashboards:
 - file: node-full
   datasource: "Prometheus"
 - file: openwrt
   datasource: "Prometheus"
 - file: chrony
   datasource: "Prometheus"
 - file: environment-sensors
   datasource: "Prometheus"
 - file: blackbox/ssh
   datasource: "Prometheus"
 - file: blackbox/https
   datasource: "Prometheus"
 - file: blackbox/mqtt
   datasource: "Prometheus"
 - file: smokeping
   datasource: "Prometheus"
 - file: bind
   datasource: "Prometheus"
 - file: ipmi
   datasource: "Prometheus"
 - file: standalone-kubelet-overview
   datasource: "Prometheus"
 - file: apps/whawty-nginx-sso
   datasource: "Prometheus"
 - file: mosquitto
   datasource: "Prometheus"

grafana_admin_password: "{{ vault_grafana_admin_password }}"



monitoring_landingpage_hostnames:
  - "mon.chaos-at-home.org"
monitoring_landingpage_title: "chaos@home Monitoring Host"
monitoring_landingpage_tls:
  certificate_provider: static-ca
  certificate_config:
    mode: "0750"
    owner: root
    group: www-data
    ca:
      key_content: "{{ chaos_at_home_internal_ca_key }}"
      cert_content: "{{ chaos_at_home_internal_ca_cert }}"
    key:
      mode: "0640"
      owner: root
      group: www-data
      type: RSA
      size: 4096
    cert:
      mode: "0644"
      owner: root
      group: www-data
      common_name: "{{ host_name }}"
      san_extra: "{{ ['IP:'] | product(ansible_all_ipv4_addresses) | map('join') | list }}"
      key_usage:
      - digitalSignature
      - keyAgreement
      key_usage_critical: yes
      extended_key_usage:
      - serverAuth
      extended_key_usage_critical: yes
      create_subject_key_identifier: yes
      not_before: +0h
      not_after: +365d
      renew_margin: +70d

monitoring_landingpage_vhost_extra_directives: |
  include snippets/whawty-sso-chaos-at-home.conf;

  location = /healthz {
     auth_request off;
     return 200;
  }

monitoring_landingpage_service_extra_directives:
  prometheus: |
    proxy_set_header Authorization "Basic {{ ('proxy:'~prometheus_server_auth_users['proxy']) | b64encode }}";
  alertmanager: |
    proxy_set_header Authorization "Basic {{ ('proxy:'~prometheus_alertmanager_auth_users['proxy']) | b64encode }}";
  grafana: |
    auth_request_set $username $upstream_http_x_username;
    proxy_set_header X-WEBAUTH-USER $username;
    proxy_set_header Authorization "";