--- install_jumphost: ch-jump system_lvm_volume_size_root: 3G install: vm: memory: 8G numcpus: 8 autostart: yes disks: primary: /dev/sda scsi: sda: type: zfs name: root size: 10g sdb: type: zfs name: data size: 50g interfaces: - bridge: br-svc name: svc0 - bridge: br-iot name: iot0 - bridge: br-mgmt name: mgmt0 network: nameservers: "{{ network_zones.svc.dns }}" domain: "{{ host_domain }}" systemd_link: interfaces: "{{ install.interfaces }}" primary: &_network_primary_ name: svc0 address: "{{ network_zones.svc.prefix | ansible.utils.ipaddr(network_zones.svc.offsets[inventory_hostname]) }}" gateway: "{{ network_zones.svc.gateway }}" static_routes: - destination: "{{ network_zones.lan.prefix }}" gateway: "{{ network_zones.svc.prefix | ansible.utils.ipaddr(network_zones.svc.offsets['ch-gw-lan']) | ansible.utils.ipaddr('address') }}" interfaces: - *_network_primary_ - name: iot0 address: "{{ network_zones.iot.prefix | ansible.utils.ipaddr(network_zones.iot.offsets[inventory_hostname]) }}" - name: mgmt0 address: "{{ network_zones.mgmt.prefix | ansible.utils.ipaddr(network_zones.mgmt.offsets[inventory_hostname]) }}" lvm_groups: mondata: pvs: - /dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_drive-scsi0-0-0-1 spreadspace_apt_repo_components: - main - prometheus nftables_base_rules: main: | table inet global { chain input_iot { ip saddr != {{ network_zones.iot.prefix }} drop ip protocol icmp accept ip6 nexthdr ipv6-icmp accept } chain input_mgmt { ip saddr != {{ network_zones.mgmt.prefix }} drop ip protocol icmp accept ip6 nexthdr ipv6-icmp accept } chain input { type filter hook input priority filter; policy drop; ct state vmap { established: accept, related: accept, invalid: drop } iifname vmap { lo: accept, svc0: accept, iot0: jump input_iot, mgmt0: jump input_mgmt } } chain forward { type filter hook forward priority 0; policy drop; } } protect-grafana-auth-proxy: | table inet filter { chain protect-grafana-auth-proxy { type filter hook output priority filter; policy accept; meta skuid != { root, www-data } tcp dport 3000 counter reject } } whawty_nginx_sso_backends: chaos-at-home: port: 1234 login_url: https://login.chaos-at-home.org/login whawty_nginx_sso_auths: chaos-at-home: config: cookie: name: __Secure-chaos-at-home-sso keys: - name: 2023-11 ed25519: public-key-data: |- -----BEGIN PUBLIC KEY----- MCowBQYDK2VwAyEAawvVwThGnYYBDLjQ0Rs71prAmxQ/tfaPUNZvPWS3Z3U= -----END PUBLIC KEY----- backend: bolt: {} sync: base-url: "https://{{ network_services.http.addr }}" http-host: "login.chaos-at-home.org" token: "{{ vault_whawty_nginx_sso_sync_tokens['chaos-at-home'][inventory_hostname] }}" tls: server-name: "login.chaos-at-home.org" web: listen: 127.0.0.1:1234 prometheus: {} prometheus_job_multitarget_whawty_nginx_sso: ch-mon: - app_instance: chaos-at-home prometheus_server_storage: type: lvm vg: mondata lv: prometheus size: 30G fs: ext4 prometheus_server_external_labels: environment: chaos-at-home monitor: "{{ inventory_hostname }}" prometheus_server_alertmanager: url: "127.0.0.1:9093" path_prefix: "/alertmanager/" basic_auth: username: server password: "{{ vault_prometheus_alertmanager_auth_user_passwords['server'] }}" prometheus_server_web_external_url: "http://mon.chaos-at-home.org/prometheus/" prometheus_server_auth_users: server: "{{ vault_prometheus_server_auth_user_passwords['server'] }}" grafana: "{{ vault_prometheus_server_auth_user_passwords['grafana'] }}" proxy: "{{ vault_prometheus_server_auth_user_passwords['proxy'] }}" prometheus_server_selfscraping_auth: username: server password: "{{ vault_prometheus_server_auth_user_passwords['server'] }}" prometheus_exporters_extra: - blackbox - nut - ssl - smokeping prometheus_exporter_smokeping_targets: - hosts: - "{{ network_zones.magenta.gateway }}" - "{{ network_zones.magenta.dns[0] }}" - "{{ network_zones.magenta.dns[1] }}" - 9.9.9.9 - 8.8.8.8 - 1.1.1.1 - "{{ hostvars['ch-atlas'].vm_host.network.bridges.public.prefix | ansible.utils.ipaddr(hostvars['ch-atlas'].vm_host.network.bridges.public.offsets['ch-atlas']) | ansible.utils.ipaddr('address') }}" prometheus_job_multitarget_blackbox__probe: ch-mon: - svc_kind: ssh svc_instance: "{{ inventory_hostname }}" target: "{{ network_zones.svc.prefix | ansible.utils.ipaddr(network_zones.svc.offsets[inventory_hostname]) | ansible.utils.ipaddr('address') }}:{{ ansible_port | default(22) }}" module: ssh_banner - svc_kind: https svc_instance: "mon.chaos-at-home.org" target: "https://{{ network_zones.svc.prefix | ansible.utils.ipaddr(network_zones.svc.offsets[inventory_hostname]) | ansible.utils.ipaddr('address') }}/healthz" module: http_tls_2xx prometheus_job_multitarget_ssl__probe: ch-mon: - module: file target: "/etc/ssl/prometheus/**/*.pem" sslcert_instance: prometheus prometheus_server_rules_node_extra: - alert: GitFsckMetricsOutdated expr: time() - git_fsck_run > 100000 for: 0m labels: severity: warning annotations: summary: Metrics from git-fsck are too old (instance {{ '{{' }} $labels.instance {{ '}}' }}) description: "The exported values from git-fsck on host {{ '{{' }} $labels.instance {{ '}}' }} are {{ '{{' }} $value {{ '}}' }} seconds old.\n VALUE = {{ '{{' }} $value {{ '}}' }}\n LABELS = {{ '{{' }} $labels {{ '}}' }}" - alert: GitFsckFailed expr: git_fsck_failed != 0 for: 0m labels: severity: warning annotations: summary: git-filesystem check failed (instance {{ '{{' }} $labels.instance {{ '}}' }}) description: "The git repostory {{ '{{' }} $labels.repository {{ '}}' }}@{{ '{{' }} $labels.gitolite_instance {{ '}}' }} on host {{ '{{' }} $labels.instance {{ '}}' }} is corrupt.\n VALUE = {{ '{{' }} $value {{ '}}' }}\n LABELS = {{ '{{' }} $labels {{ '}}' }}" prometheus_alertmanager_smtp: smarthost: "{{ network_zones.lan.prefix | ansible.utils.ipaddr(network_zones.lan.offsets['ch-prometheus-legacy']) | ansible.utils.ipaddr('address') }}:25" from: "noreply@chaos-at-home.org" require_tls: no prometheus_alertmanager_web_external_url: "http://mon.chaos-at-home.org/alertmanager/" prometheus_alertmanager_auth_users: server: "{{ vault_prometheus_alertmanager_auth_user_passwords['server'] }}" proxy: "{{ vault_prometheus_alertmanager_auth_user_passwords['proxy'] }}" prometheus_alertmanager_route: receiver: empty routes: - receiver: equinox-mail matchers: - 'alertname != PrometheusAlertmanagerE2eDeadManSwitch' group_by: - instance prometheus_alertmanager_receivers: - name: empty - name: equinox-mail email_configs: - to: equinox@chaos-at-home.org send_resolved: yes grafana_secret_key: "{{ vault_grafana_secret_key }}" grafana_config_auth: disable_signout_menu: true grafana_config_auth_proxy: enabled: true whitelist: 127.0.0.1 grafana_datasources: - name: "Prometheus" type: "prometheus" access: "proxy" url: "http://127.0.0.1:9090/prometheus" basicAuth: true basicAuthUser: "grafana" isDefault: yes secureJsonData: basicAuthPassword: "{{ vault_prometheus_server_auth_user_passwords['grafana'] }}" jsonData: manageAlerts: no grafana_dashboards: - file: node-full datasource: "Prometheus" - file: openwrt datasource: "Prometheus" - file: chrony datasource: "Prometheus" - file: environment-sensors datasource: "Prometheus" - file: blackbox/ssh datasource: "Prometheus" - file: blackbox/https datasource: "Prometheus" - file: blackbox/mqtt datasource: "Prometheus" - file: smokeping datasource: "Prometheus" - file: bind datasource: "Prometheus" - file: ipmi datasource: "Prometheus" - file: standalone-kubelet-overview datasource: "Prometheus" - file: apps/whawty-nginx-sso datasource: "Prometheus" - file: mosquitto datasource: "Prometheus" grafana_admin_password: "{{ vault_grafana_admin_password }}" monitoring_landingpage_hostnames: - "mon.chaos-at-home.org" monitoring_landingpage_title: "chaos@home Monitoring Host" monitoring_landingpage_tls: certificate_provider: static-ca certificate_config: mode: "0750" owner: root group: www-data ca: key_content: "{{ chaos_at_home_internal_ca_key }}" cert_content: "{{ chaos_at_home_internal_ca_cert }}" key: mode: "0640" owner: root group: www-data type: RSA size: 4096 cert: mode: "0644" owner: root group: www-data common_name: "{{ host_name }}" san_extra: "{{ ['IP:'] | product(ansible_all_ipv4_addresses) | map('join') | list }}" key_usage: - digitalSignature - keyAgreement key_usage_critical: yes extended_key_usage: - serverAuth extended_key_usage_critical: yes create_subject_key_identifier: yes not_before: +0h not_after: +365d renew_margin: +70d monitoring_landingpage_vhost_extra_directives: | include snippets/whawty-sso-chaos-at-home.conf; location = /healthz { auth_request off; return 200; } monitoring_landingpage_service_extra_directives: prometheus: | proxy_set_header Authorization "Basic {{ ('proxy:'~prometheus_server_auth_users['proxy']) | b64encode }}"; alertmanager: | proxy_set_header Authorization "Basic {{ ('proxy:'~prometheus_alertmanager_auth_users['proxy']) | b64encode }}"; grafana: | auth_request_set $username $upstream_http_x_username; proxy_set_header X-WEBAUTH-USER $username; proxy_set_header Authorization "";