diff options
Diffstat (limited to 'roles/monitoring/prometheus/exporter')
33 files changed, 852 insertions, 85 deletions
diff --git a/roles/monitoring/prometheus/exporter/TODO b/roles/monitoring/prometheus/exporter/TODO index c02e5699..57179464 100644 --- a/roles/monitoring/prometheus/exporter/TODO +++ b/roles/monitoring/prometheus/exporter/TODO @@ -1,11 +1,3 @@ -Node Exporter - Text Collector Scripts: - - https://github.com/prometheus-community/node-exporter-textfile-collector-scripts - - https://packages.debian.org/bullseye/prometheus-node-exporter-collectors - -IPMI Exporter: - - https://github.com/soundcloud/ipmi_exporter - - https://packages.debian.org/bullseye/prometheus-ipmi-exporter - Postfix Exporter: - https://github.com/kumina/postfix_exporter - https://packages.debian.org/bullseye/prometheus-postfix-exporter diff --git a/roles/monitoring/prometheus/exporter/base/defaults/main.yml b/roles/monitoring/prometheus/exporter/base/defaults/main.yml index 963763a5..613943d8 100644 --- a/roles/monitoring/prometheus/exporter/base/defaults/main.yml +++ b/roles/monitoring/prometheus/exporter/base/defaults/main.yml @@ -1,2 +1,2 @@ --- -prometheus_exporter_listen: ":9999" +prometheus_exporter_listen: "9999" diff --git a/roles/monitoring/prometheus/exporter/base/handlers/main.yml b/roles/monitoring/prometheus/exporter/base/handlers/main.yml index ebd760cf..d4e42ca0 100644 --- a/roles/monitoring/prometheus/exporter/base/handlers/main.yml +++ b/roles/monitoring/prometheus/exporter/base/handlers/main.yml @@ -1,5 +1,5 @@ --- -- name: restart prometheus-exporter-exporter +- name: reload nginx service: - name: prometheus-exporter-exporter - state: restarted + name: nginx + state: reloaded diff --git a/roles/monitoring/prometheus/exporter/base/tasks/main.yml b/roles/monitoring/prometheus/exporter/base/tasks/main.yml index 9a214f39..5f42867d 100644 --- a/roles/monitoring/prometheus/exporter/base/tasks/main.yml +++ b/roles/monitoring/prometheus/exporter/base/tasks/main.yml @@ -6,16 +6,6 @@ - spreadspace_apt_repo_components is defined - "'prometheus' in spreadspace_apt_repo_components" -- name: install apt packages - apt: - name: prom-exporter-exporter - state: present - -- name: create configuration directories - file: - path: /etc/prometheus/exporter/enabled - state: directory - - name: add user for prometheus-exporter user: name: prometheus-exporter @@ -26,15 +16,10 @@ - name: create TLS certificate and key import_tasks: tls.yml -- name: generate systemd service unit - template: - src: service.j2 - dest: /etc/systemd/system/prometheus-exporter-exporter.service - notify: restart prometheus-exporter-exporter - -- name: make sure prometheus-exporter-exporter is enabled and started - systemd: - name: prometheus-exporter-exporter.service - daemon_reload: yes - state: started - enabled: yes +- name: configure nginx vhost + import_role: + name: nginx/vhost + vars: + nginx_vhost: + name: prometheus-exporter + content: "{{ lookup('template', 'nginx-vhost.j2') }}" diff --git a/roles/monitoring/prometheus/exporter/base/templates/nginx-vhost.j2 b/roles/monitoring/prometheus/exporter/base/templates/nginx-vhost.j2 new file mode 100644 index 00000000..70e65b29 --- /dev/null +++ b/roles/monitoring/prometheus/exporter/base/templates/nginx-vhost.j2 @@ -0,0 +1,19 @@ +server { + listen {{ prometheus_exporter_listen }} ssl; + server_name _; + + ssl_certificate /etc/ssl/prometheus/exporter/crt.pem; + ssl_certificate_key /etc/ssl/prometheus/exporter/key.pem; + ssl_client_certificate /etc/ssl/prometheus/ca-crt.pem; + ssl_verify_client on; + + root /nonexistent; + + location = / { + return 404 'please specify the exporter you want to reach!'; + } + + include snippets/proxy-nobuff.conf; + + include /etc/prometheus/exporter/*.locations; +} diff --git a/roles/monitoring/prometheus/exporter/blackbox/defaults/main.yml b/roles/monitoring/prometheus/exporter/blackbox/defaults/main.yml index 4e7d8d9a..73b9fde1 100644 --- a/roles/monitoring/prometheus/exporter/blackbox/defaults/main.yml +++ b/roles/monitoring/prometheus/exporter/blackbox/defaults/main.yml @@ -2,23 +2,30 @@ prometheus_exporter_blackbox_modules: tcp_connect: prober: tcp + tcp: + preferred_ip_protocol: "ip4" tcp_tls_connect: prober: tcp tcp: + preferred_ip_protocol: "ip4" tls: true tls_config: insecure_skip_verify: true http_2xx: prober: http + http: + preferred_ip_protocol: "ip4" http_tls_2xx: prober: http http: + preferred_ip_protocol: "ip4" fail_if_not_ssl: true tls_config: insecure_skip_verify: true ssh_banner: prober: tcp tcp: + preferred_ip_protocol: "ip4" query_response: - expect: "^SSH-2.0-" - send: "SSH-2.0-blackbox-ssh-check" diff --git a/roles/monitoring/prometheus/exporter/blackbox/handlers/main.yml b/roles/monitoring/prometheus/exporter/blackbox/handlers/main.yml index 99a416e2..12250769 100644 --- a/roles/monitoring/prometheus/exporter/blackbox/handlers/main.yml +++ b/roles/monitoring/prometheus/exporter/blackbox/handlers/main.yml @@ -9,8 +9,7 @@ name: prometheus-blackbox-exporter state: reloaded -- name: reload prometheus-exporter-exporter +- name: reload nginx service: - name: prometheus-exporter-exporter - ## TODO: implement reload once exporter_exporter supports this... - state: restarted + name: nginx + state: reloaded diff --git a/roles/monitoring/prometheus/exporter/blackbox/tasks/main.yml b/roles/monitoring/prometheus/exporter/blackbox/tasks/main.yml index 7ecd8113..c4cabfce 100644 --- a/roles/monitoring/prometheus/exporter/blackbox/tasks/main.yml +++ b/roles/monitoring/prometheus/exporter/blackbox/tasks/main.yml @@ -1,4 +1,5 @@ --- + ## TODO: pin version - name: install apt packages apt: name: prom-exporter-blackbox @@ -31,9 +32,11 @@ - name: register exporter copy: content: | - method: http - http: - port: 9115 - path: '/probe' - dest: /etc/prometheus/exporter/enabled/blackbox.yml - notify: reload prometheus-exporter-exporter + location = /blackbox { + proxy_pass http://127.0.0.1:9115/metrics; + } + location = /blackbox/probe { + proxy_pass http://127.0.0.1:9115/probe; + } + dest: /etc/prometheus/exporter/blackbox.locations + notify: reload nginx diff --git a/roles/monitoring/prometheus/exporter/blackbox/templates/config.yml.j2 b/roles/monitoring/prometheus/exporter/blackbox/templates/config.yml.j2 index 01e3f7a0..0ff9db13 100644 --- a/roles/monitoring/prometheus/exporter/blackbox/templates/config.yml.j2 +++ b/roles/monitoring/prometheus/exporter/blackbox/templates/config.yml.j2 @@ -1,4 +1,4 @@ # {{ ansible_managed }} modules: - {{ prometheus_exporter_blackbox_modules | combine(prometheus_exporter_blackbox_modules_extra) | to_nice_yaml(indent=2) | indent(2)}} + {{ prometheus_exporter_blackbox_modules | combine(prometheus_exporter_blackbox_modules_extra) | to_nice_yaml(indent=2) | indent(2) }} diff --git a/roles/monitoring/prometheus/exporter/ipmi/defaults/main.yml b/roles/monitoring/prometheus/exporter/ipmi/defaults/main.yml new file mode 100644 index 00000000..6cf14f76 --- /dev/null +++ b/roles/monitoring/prometheus/exporter/ipmi/defaults/main.yml @@ -0,0 +1,26 @@ +--- +prometheus_exporter_ipmi_modules: + default: + collectors: [] + # collectors: + # - bmc + # - ipmi + # - chassis + # - dcmi + # - sel + # - sm-lan-mode + # exclude_sensor_ids: + # - 2 + # - 29 + # - 32 + # thatspecialhost: + # user: "some_user" + # pass: "secret_pw" + # privilege: "admin" + # driver: "LAN" + # collectors: + # - ipmi + # - sel + # custom_args: + # ipmi: + # - "--bridge-sensors" diff --git a/roles/monitoring/prometheus/exporter/ipmi/handlers/main.yml b/roles/monitoring/prometheus/exporter/ipmi/handlers/main.yml new file mode 100644 index 00000000..a8eb55b3 --- /dev/null +++ b/roles/monitoring/prometheus/exporter/ipmi/handlers/main.yml @@ -0,0 +1,15 @@ +--- +- name: restart prometheus-ipmi-exporter + service: + name: prometheus-ipmi-exporter + state: restarted + +- name: reload prometheus-ipmi-exporter + service: + name: prometheus-ipmi-exporter + state: reloaded + +- name: reload nginx + service: + name: nginx + state: reloaded diff --git a/roles/monitoring/prometheus/exporter/ipmi/tasks/main.yml b/roles/monitoring/prometheus/exporter/ipmi/tasks/main.yml new file mode 100644 index 00000000..91318f16 --- /dev/null +++ b/roles/monitoring/prometheus/exporter/ipmi/tasks/main.yml @@ -0,0 +1,42 @@ +--- + ## TODO: pin version +- name: install apt packages + apt: + name: prom-exporter-ipmi + state: present + +- name: create config directory + file: + path: /etc/prometheus/exporter/ipmi + state: directory + +- name: generate configuration + template: + src: config.yml.j2 + dest: /etc/prometheus/exporter/ipmi/config.yml + notify: reload prometheus-ipmi-exporter + +- name: generate systemd service unit + template: + src: service.j2 + dest: /etc/systemd/system/prometheus-ipmi-exporter.service + notify: restart prometheus-ipmi-exporter + +- name: make sure prometheus-ipmi-exporter is enabled and started + systemd: + name: prometheus-ipmi-exporter.service + daemon_reload: yes + state: started + enabled: yes + +- name: register exporter + copy: + content: | + location = /ipmi { + proxy_pass http://127.0.0.1:9290/metrics; + } + location = /ipmi/remote { + proxy_pass http://127.0.0.1:9290/ipmi; + } + dest: /etc/prometheus/exporter/ipmi.locations + notify: reload nginx diff --git a/roles/monitoring/prometheus/exporter/ipmi/templates/config.yml.j2 b/roles/monitoring/prometheus/exporter/ipmi/templates/config.yml.j2 new file mode 100644 index 00000000..32d0b34a --- /dev/null +++ b/roles/monitoring/prometheus/exporter/ipmi/templates/config.yml.j2 @@ -0,0 +1,4 @@ +# {{ ansible_managed }} + +modules: + {{ prometheus_exporter_ipmi_modules | to_nice_yaml(indent=2) | indent(2) }} diff --git a/roles/monitoring/prometheus/exporter/base/templates/service.j2 b/roles/monitoring/prometheus/exporter/ipmi/templates/service.j2 index c24baf43..465215e8 100644 --- a/roles/monitoring/prometheus/exporter/base/templates/service.j2 +++ b/roles/monitoring/prometheus/exporter/ipmi/templates/service.j2 @@ -1,12 +1,13 @@ [Unit] -Description=Prometheus exporter proxy +Description=Prometheus ipmi exporter [Service] Restart=always User=prometheus-exporter -ExecStart=/usr/bin/prometheus-exporter-exporter -config.dirs=/etc/prometheus/exporter/enabled -config.file="" -web.listen-address="" -web.tls.listen-address="{{ prometheus_exporter_listen }}" -web.tls.cert="/etc/ssl/prometheus/exporter/crt.pem" -web.tls.key="/etc/ssl/prometheus/exporter/key.pem" --web.tls.ca="/etc/ssl/prometheus/ca-crt.pem" -web.tls.verify -{# TODO: implement reloading once the exporter_exporter supports this #} +ExecStart=/usr/bin/prometheus-ipmi-exporter --web.listen-address="127.0.0.1:9290" --config.file=/etc/prometheus/exporter/ipmi/config.yml --freeipmi.path="/usr/sbin" +ExecReload=/bin/kill -HUP $MAINPID +{# TODO: test which hardening options need to be removed for IPMI to work... #} # systemd hardening-options AmbientCapabilities= CapabilityBoundingSet= @@ -17,7 +18,6 @@ MemoryDenyWriteExecute=true NoNewPrivileges=true PrivateDevices=true PrivateTmp=true -PrivateUsers=true ProtectControlGroups=true ProtectHome=true ProtectKernelModules=true diff --git a/roles/monitoring/prometheus/exporter/meta/main.yml b/roles/monitoring/prometheus/exporter/meta/main.yml index 22131422..68fce6cb 100644 --- a/roles/monitoring/prometheus/exporter/meta/main.yml +++ b/roles/monitoring/prometheus/exporter/meta/main.yml @@ -1,11 +1,13 @@ --- dependencies: - role: monitoring/prometheus/exporter/base - - role: monitoring/prometheus/exporter/node - when: "'node' in (prometheus_exporters_default | union(prometheus_exporters_extra))" - role: monitoring/prometheus/exporter/blackbox when: "'blackbox' in (prometheus_exporters_default | union(prometheus_exporters_extra))" - - role: monitoring/prometheus/exporter/nut - when: "'nut' in (prometheus_exporters_default | union(prometheus_exporters_extra))" + - role: monitoring/prometheus/exporter/ipmi + when: "'ipmi' in (prometheus_exporters_default | union(prometheus_exporters_extra))" - role: monitoring/prometheus/exporter/mikrotik when: "'mikrotik' in (prometheus_exporters_default | union(prometheus_exporters_extra))" + - role: monitoring/prometheus/exporter/node + when: "'node' in (prometheus_exporters_default | union(prometheus_exporters_extra))" + - role: monitoring/prometheus/exporter/nut + when: "'nut' in (prometheus_exporters_default | union(prometheus_exporters_extra))" diff --git a/roles/monitoring/prometheus/exporter/mikrotik/handlers/main.yml b/roles/monitoring/prometheus/exporter/mikrotik/handlers/main.yml index cb85d0d9..c5844220 100644 --- a/roles/monitoring/prometheus/exporter/mikrotik/handlers/main.yml +++ b/roles/monitoring/prometheus/exporter/mikrotik/handlers/main.yml @@ -4,8 +4,7 @@ name: prometheus-mikrotik-exporter state: restarted -- name: reload prometheus-exporter-exporter +- name: reload nginx service: - name: prometheus-exporter-exporter - ## TODO: implement reload once exporter_exporter supports this... - state: restarted + name: nginx + state: reloaded diff --git a/roles/monitoring/prometheus/exporter/mikrotik/tasks/main.yml b/roles/monitoring/prometheus/exporter/mikrotik/tasks/main.yml index c3ffe31b..72c78e4a 100644 --- a/roles/monitoring/prometheus/exporter/mikrotik/tasks/main.yml +++ b/roles/monitoring/prometheus/exporter/mikrotik/tasks/main.yml @@ -1,4 +1,5 @@ --- + ## TODO: pin version - name: install apt packages apt: name: prom-exporter-mikrotik @@ -34,8 +35,8 @@ - name: register exporter copy: content: | - method: http - http: - port: 9436 - dest: /etc/prometheus/exporter/enabled/mikrotik.yml - notify: reload prometheus-exporter-exporter + location = /mikrotik { + proxy_pass http://127.0.0.1:9436/metrics; + } + dest: /etc/prometheus/exporter/mikrotik.locations + notify: reload nginx diff --git a/roles/monitoring/prometheus/exporter/mikrotik/templates/config.yml.j2 b/roles/monitoring/prometheus/exporter/mikrotik/templates/config.yml.j2 index a2dc1c71..576ee12f 100644 --- a/roles/monitoring/prometheus/exporter/mikrotik/templates/config.yml.j2 +++ b/roles/monitoring/prometheus/exporter/mikrotik/templates/config.yml.j2 @@ -1,7 +1,7 @@ # {{ ansible_managed }} devices: - {{ prometheus_exporter_mikrotik_devices | to_nice_yaml(indent=2) | indent(2)}} + {{ prometheus_exporter_mikrotik_devices | to_nice_yaml(indent=2) | indent(2) }} features: - {{ prometheus_exporter_mikrotik_features | to_nice_yaml(indent=2) | indent(2)}} + {{ prometheus_exporter_mikrotik_features | to_nice_yaml(indent=2) | indent(2) }} diff --git a/roles/monitoring/prometheus/exporter/node/defaults/main.yml b/roles/monitoring/prometheus/exporter/node/defaults/main.yml index 56227fbb..9309562f 100644 --- a/roles/monitoring/prometheus/exporter/node/defaults/main.yml +++ b/roles/monitoring/prometheus/exporter/node/defaults/main.yml @@ -9,4 +9,8 @@ prometheus_exporter_node_timesync_collector: "{{ _prometheus_exporter_node_time_ prometheus_exporter_node_disable_collectors: [] prometheus_exporter_node_extra_collectors: -- "{{ prometheus_exporter_node_timesync_collector }}" + - "{{ prometheus_exporter_node_timesync_collector }}" + - systemd + +prometheus_exporter_node_textfile_collector_scripts: + - deleted-libraries diff --git a/roles/monitoring/prometheus/exporter/node/files/apt b/roles/monitoring/prometheus/exporter/node/files/apt new file mode 100755 index 00000000..015addb0 --- /dev/null +++ b/roles/monitoring/prometheus/exporter/node/files/apt @@ -0,0 +1,40 @@ +#!/bin/bash +# +# Description: Expose metrics from apt updates. +# +# Author: Ben Kochie <superq@gmail.com> + +upgrades="$(/usr/bin/apt-get --just-print dist-upgrade \ + | /usr/bin/awk -F'[()]' \ + '/^Inst/ { sub("^[^ ]+ ", "", $2); gsub(" ","",$2); + sub("\\[", " ", $2); sub("\\]", "", $2); print $2 }' \ + | /usr/bin/sort \ + | /usr/bin/uniq -c \ + | awk '{ gsub(/\\\\/, "\\\\", $2); gsub(/"/, "\\\"", $2); + gsub(/\[/, "", $3); gsub(/\]/, "", $3); + print "apt_upgrades_pending{origin=\"" $2 "\",arch=\"" $NF "\"} " $1}' +)" + +autoremove="$(/usr/bin/apt-get --just-print autoremove \ + | /usr/bin/awk '/^Remv/{a++}END{printf "apt_autoremove_pending %d", a}' +)" + +echo '# HELP apt_upgrades_pending Apt package pending updates by origin.' +echo '# TYPE apt_upgrades_pending gauge' +if [[ -n "${upgrades}" ]] ; then + echo "${upgrades}" +else + echo 'apt_upgrades_pending{origin="",arch=""} 0' +fi + +echo '# HELP apt_autoremove_pending Apt package pending autoremove.' +echo '# TYPE apt_autoremove_pending gauge' +echo "${autoremove}" + +echo '# HELP node_reboot_required Node reboot is required for software updates.' +echo '# TYPE node_reboot_required gauge' +if [[ -f '/run/reboot-required' ]] ; then + echo 'node_reboot_required 1' +else + echo 'node_reboot_required 0' +fi diff --git a/roles/monitoring/prometheus/exporter/node/files/deleted-libraries b/roles/monitoring/prometheus/exporter/node/files/deleted-libraries new file mode 100755 index 00000000..e3e19cbd --- /dev/null +++ b/roles/monitoring/prometheus/exporter/node/files/deleted-libraries @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +""" +Script to count the number of deleted libraries that are linked by running +processes and expose a summary as Prometheus metrics. + +The aim is to discover processes that are still using libraries that have since +been updated, perhaps due security vulnerabilities. +""" + +import errno +import glob +import os +import sys + + +def main(): + processes_linking_deleted_libraries = {} + + for path in glob.glob('/proc/*/maps'): + try: + with open(path, 'rb') as file: + for line in file: + part = line.decode().strip().split() + + if len(part) == 7: + library = part[5] + comment = part[6] + + if '/lib/' in library and '(deleted)' in comment: + if path not in processes_linking_deleted_libraries: + processes_linking_deleted_libraries[path] = {} + + if library in processes_linking_deleted_libraries[path]: + processes_linking_deleted_libraries[path][library] += 1 + else: + processes_linking_deleted_libraries[path][library] = 1 + except EnvironmentError as e: + # Ignore non-existent files, since the files may have changed since + # we globbed. + if e.errno != errno.ENOENT: + sys.exit('Failed to open file: {0}'.format(path)) + + num_processes_per_library = {} + + for process, library_count in processes_linking_deleted_libraries.items(): + libraries_seen = set() + for library, count in library_count.items(): + if library in libraries_seen: + continue + + libraries_seen.add(library) + if library in num_processes_per_library: + num_processes_per_library[library] += 1 + else: + num_processes_per_library[library] = 1 + + metric_name = 'node_processes_linking_deleted_libraries' + description = 'Count of running processes that link a deleted library' + print('# HELP {0} {1}'.format(metric_name, description)) + print('# TYPE {0} gauge'.format(metric_name)) + + for library, count in num_processes_per_library.items(): + dir_path, basename = os.path.split(library) + basename = basename.replace('"', '\\"') + dir_path = dir_path.replace('"', '\\"') + print('{0}{{library_path="{1}", library_name="{2}"}} {3}'.format( + metric_name, + dir_path, + basename, + count) + ) + + +if __name__ == "__main__": + main() diff --git a/roles/monitoring/prometheus/exporter/node/files/smartmon b/roles/monitoring/prometheus/exporter/node/files/smartmon new file mode 100644 index 00000000..1c39b492 --- /dev/null +++ b/roles/monitoring/prometheus/exporter/node/files/smartmon @@ -0,0 +1,391 @@ +#!/usr/bin/env python3 +import argparse +import collections +import csv +import datetime +import decimal +import re +import shlex +import subprocess +import sys + +device_info_re = re.compile(r'^(?P<k>[^:]+?)(?:(?:\sis|):)\s*(?P<v>.*)$') + +ata_error_count_re = re.compile( + r'^Error (\d+) \[\d+\] occurred', re.MULTILINE) + +self_test_re = re.compile(r'^SMART.*(PASSED|OK)$', re.MULTILINE) + +device_info_map = { + 'Vendor': 'vendor', + 'Product': 'product', + 'Revision': 'revision', + 'Logical Unit id': 'lun_id', + 'Model Family': 'model_family', + 'Device Model': 'device_model', + 'Serial Number': 'serial_number', + 'Firmware Version': 'firmware_version', +} + +smart_attributes_whitelist = { + 'airflow_temperature_cel', + 'command_timeout', + 'current_pending_sector', + 'end_to_end_error', + 'erase_fail_count_total', + 'g_sense_error_rate', + 'hardware_ecc_recovered', + 'host_reads_mib', + 'host_reads_32mib', + 'host_writes_mib', + 'host_writes_32mib', + 'load_cycle_count', + 'media_wearout_indicator', + 'wear_leveling_count', + 'nand_writes_1gib', + 'offline_uncorrectable', + 'power_cycle_count', + 'power_on_hours', + 'program_fail_count', + 'raw_read_error_rate', + 'reallocated_event_count', + 'reallocated_sector_ct', + 'reported_uncorrect', + 'sata_downshift_count', + 'seek_error_rate', + 'spin_retry_count', + 'spin_up_time', + 'start_stop_count', + 'temperature_case', + 'temperature_celsius', + 'temperature_internal', + 'total_lbas_read', + 'total_lbas_written', + 'udma_crc_error_count', + 'unsafe_shutdown_count', + 'workld_host_reads_perc', + 'workld_media_wear_indic', + 'workload_minutes', +} + +Metric = collections.namedtuple('Metric', 'name labels value') + +SmartAttribute = collections.namedtuple('SmartAttribute', [ + 'id', 'name', 'flag', 'value', 'worst', 'threshold', 'type', 'updated', + 'when_failed', 'raw_value', +]) + + +class Device(collections.namedtuple('DeviceBase', 'path opts')): + """Representation of a device as found by smartctl --scan output.""" + + @property + def type(self): + return self.opts.type + + @property + def base_labels(self): + return {'device': self.path, 'disk': self.type.partition('+')[2] or '0'} + + def smartctl_select(self): + return ['--device', self.type, self.path] + + +def metric_key(metric, prefix=''): + return '{prefix}{metric.name}'.format(prefix=prefix, metric=metric) + + +def metric_format(metric, prefix=''): + key = metric_key(metric, prefix) + labels = ','.join( + '{k}="{v}"'.format(k=k, v=v.replace('"', '\\"')) for k, v in metric.labels.items()) + value = decimal.Decimal(metric.value) + + return '{key}{{{labels}}} {value}'.format( + key=key, labels=labels, value=value) + + +def metric_print_meta(metric, prefix=''): + key = metric_key(metric, prefix) + print('# HELP {key} SMART metric {metric.name}'.format( + key=key, metric=metric)) + print('# TYPE {key} gauge'.format(key=key)) + + +def metric_print(metric, prefix=''): + print(metric_format(metric, prefix)) + + +def smart_ctl(*args, check=True): + """Wrapper around invoking the smartctl binary. + + Returns: + (str) Data piped to stdout by the smartctl subprocess. + """ + return subprocess.run( + ['smartctl', *args], stdout=subprocess.PIPE, check=check + ).stdout.decode('utf-8') + + +def smart_ctl_version(): + return smart_ctl('-V').split('\n')[0].split()[1] + + +def find_devices(): + """Find SMART devices. + + Yields: + (Device) Single device found by smartctl. + """ + parser = argparse.ArgumentParser() + parser.add_argument('-d', '--device', dest='type') + + devices = smart_ctl('--scan-open') + + for device in devices.split('\n'): + device = device.strip() + if not device: + continue + + tokens = shlex.split(device, comments=True) + if not tokens: + continue + + yield Device(tokens[0], parser.parse_args(tokens[1:])) + + +def device_is_active(device): + """Returns whenever the given device is currently active or not. + + Args: + device: (Device) Device in question. + + Returns: + (bool) True if the device is active and False otherwise. + """ + try: + smart_ctl('--nocheck', 'standby', *device.smartctl_select()) + except subprocess.CalledProcessError: + return False + + return True + + +def device_info(device): + """Query device for basic model information. + + Args: + device: (Device) Device in question. + + Returns: + (generator): Generator yielding: + + key (str): Key describing the value. + value (str): Actual value. + """ + info_lines = smart_ctl( + '--info', *device.smartctl_select() + ).strip().split('\n')[3:] + + matches = (device_info_re.match(line) for line in info_lines) + return (m.groups() for m in matches if m is not None) + + +def device_smart_capabilities(device): + """Returns SMART capabilities of the given device. + + Args: + device: (Device) Device in question. + + Returns: + (tuple): tuple containing: + + (bool): True whenever SMART is available, False otherwise. + (bool): True whenever SMART is enabled, False otherwise. + """ + groups = device_info(device) + + state = { + g[1].split(' ', 1)[0] + for g in groups if g[0] == 'SMART support'} + + smart_available = 'Available' in state + smart_enabled = 'Enabled' in state + + return smart_available, smart_enabled + + +def collect_device_info(device): + """Collect basic device information. + + Args: + device: (Device) Device in question. + + Yields: + (Metric) metrics describing general device information. + """ + values = dict(device_info(device)) + yield Metric('device_info', { + **device.base_labels, + **{v: values[k] for k, v in device_info_map.items() if k in values} + }, True) + + +def collect_device_health_self_assessment(device): + """Collect metric about the device health self assessment. + + Args: + device: (Device) Device in question. + + Yields: + (Metric) Device health self assessment. + """ + out = smart_ctl('--health', *device.smartctl_select(), check=False) + + self_assessment_passed = bool(self_test_re.search(out)) + + yield Metric( + 'device_smart_healthy', device.base_labels, self_assessment_passed) + + +def collect_ata_metrics(device): + # Fetch SMART attributes for the given device. + attributes = smart_ctl( + '--attributes', *device.smartctl_select() + ) + + # replace multiple occurrences of whitespace with a single whitespace + # so that the CSV Parser recognizes individual columns properly. + attributes = re.sub(r'[\t\x20]+', ' ', attributes) + + # Turn smartctl output into a list of lines and skip to the table of + # SMART attributes. + attribute_lines = attributes.strip().split('\n')[7:] + + # Some attributes have multiple IDs but have the same name. Don't + # yield attributes that already have been reported before. + seen = set() + + reader = csv.DictReader( + (line.strip() for line in attribute_lines), + fieldnames=SmartAttribute._fields[:-1], + restkey=SmartAttribute._fields[-1], delimiter=' ') + for entry in reader: + # We're only interested in the SMART attributes that are + # whitelisted here. + entry['name'] = entry['name'].lower() + if entry['name'] not in smart_attributes_whitelist: + continue + + # Ensure that only the numeric parts are fetched from the raw_value. + # Attributes such as 194 Temperature_Celsius reported by my SSD + # are in the format of "36 (Min/Max 24/40)" which can't be expressed + # properly as a prometheus metric. + m = re.match(r'^(\d+)', ' '.join(entry['raw_value'])) + if not m: + continue + entry['raw_value'] = m.group(1) + + # Some device models report "---" in the threshold value where most + # devices would report "000". We do the substitution here because + # downstream code expects values to be convertable to integer. + if entry['threshold'] == '---': + entry['threshold'] = '0' + + if entry['name'] in smart_attributes_whitelist and entry['name'] not in seen: + labels = { + 'name': entry['name'], + **device.base_labels, + } + + for col in 'value', 'worst', 'threshold', 'raw_value': + yield Metric( + 'attr_{col}'.format(col=col), + labels, entry[col]) + + seen.add(entry['name']) + + +def collect_ata_error_count(device): + """Inspect the device error log and report the amount of entries. + + Args: + device: (Device) Device in question. + + Yields: + (Metric) Device error count. + """ + error_log = smart_ctl( + '-l', 'xerror,1', *device.smartctl_select(), check=False) + + m = ata_error_count_re.search(error_log) + + error_count = m.group(1) if m is not None else 0 + + yield Metric('device_errors', device.base_labels, error_count) + + +def collect_disks_smart_metrics(wakeup_disks): + now = int(datetime.datetime.utcnow().timestamp()) + + for device in find_devices(): + yield Metric('smartctl_run', device.base_labels, now) + + is_active = device_is_active(device) + + yield Metric('device_active', device.base_labels, is_active) + + # Skip further metrics collection to prevent the disk from + # spinning up. + if not is_active and not wakeup_disks: + continue + + yield from collect_device_info(device) + + smart_available, smart_enabled = device_smart_capabilities(device) + + yield Metric( + 'device_smart_available', device.base_labels, smart_available) + yield Metric( + 'device_smart_enabled', device.base_labels, smart_enabled) + + # Skip further metrics collection here if SMART is disabled + # on the device. Further smartctl invocations would fail + # anyways. + if not smart_available: + continue + + yield from collect_device_health_self_assessment(device) + + if device.type.startswith('sat'): + yield from collect_ata_metrics(device) + + yield from collect_ata_error_count(device) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('-s', '--wakeup-disks', dest='wakeup_disks', action='store_true') + args = parser.parse_args(sys.argv[1:]) + + version_metric = Metric('smartctl_version', { + 'version': smart_ctl_version() + }, True) + metric_print_meta(version_metric, 'smartmon_') + metric_print(version_metric, 'smartmon_') + + metrics = list(collect_disks_smart_metrics(args.wakeup_disks)) + metrics.sort(key=lambda i: i.name) + + previous_name = None + for m in metrics: + if m.name != previous_name: + metric_print_meta(m, 'smartmon_') + + previous_name = m.name + + metric_print(m, 'smartmon_') + + +if __name__ == '__main__': + main() diff --git a/roles/monitoring/prometheus/exporter/node/handlers/main.yml b/roles/monitoring/prometheus/exporter/node/handlers/main.yml index 3e1b2000..56056ea6 100644 --- a/roles/monitoring/prometheus/exporter/node/handlers/main.yml +++ b/roles/monitoring/prometheus/exporter/node/handlers/main.yml @@ -4,8 +4,7 @@ name: prometheus-node-exporter state: restarted -- name: reload prometheus-exporter-exporter +- name: reload nginx service: - name: prometheus-exporter-exporter - ## TODO: implement reload once exporter_exporter supports this... - state: restarted + name: nginx + state: reloaded diff --git a/roles/monitoring/prometheus/exporter/node/tasks/main.yml b/roles/monitoring/prometheus/exporter/node/tasks/main.yml index 8392e580..2811c759 100644 --- a/roles/monitoring/prometheus/exporter/node/tasks/main.yml +++ b/roles/monitoring/prometheus/exporter/node/tasks/main.yml @@ -1,7 +1,10 @@ --- + ## TODO: pin version - name: install apt packages apt: - name: prom-exporter-node + name: + - prom-exporter-node + - moreutils state: present - name: create directory for textfile collector @@ -25,8 +28,25 @@ - name: register exporter copy: content: | - method: http - http: - port: 9100 - dest: /etc/prometheus/exporter/enabled/node.yml - notify: reload prometheus-exporter-exporter + location = /node { + proxy_pass http://127.0.0.1:9100/metrics; + } + dest: /etc/prometheus/exporter/node.locations + notify: reload nginx + +- name: create directory for textfile collector scripts + file: + path: /usr/local/share/prometheus-node-exporter + state: directory + +- name: install the apt textfile collector script + when: ansible_pkg_mgr == "apt" + vars: + textfile_collector_name: "apt" + include_tasks: textfile_collector_script.yml + +- name: install all other textfile collector scripts + loop: "{{ prometheus_exporter_node_textfile_collector_scripts }}" + loop_control: + loop_var: textfile_collector_name + include_tasks: textfile_collector_script.yml diff --git a/roles/monitoring/prometheus/exporter/node/tasks/textfile_collector_script.yml b/roles/monitoring/prometheus/exporter/node/tasks/textfile_collector_script.yml new file mode 100644 index 00000000..80390a15 --- /dev/null +++ b/roles/monitoring/prometheus/exporter/node/tasks/textfile_collector_script.yml @@ -0,0 +1,24 @@ +--- +- name: install the collector script + copy: + src: "{{ textfile_collector_name }}" + dest: "/usr/local/share/prometheus-node-exporter/{{ textfile_collector_name }}" + mode: 0755 + +- name: install systemd service units + loop: + - service + - timer + template: + src: "textfile-collector-scripts/{{ textfile_collector_name }}.{{ item }}.j2" + dest: "/etc/systemd/system/prometheus-node-exporter_{{ textfile_collector_name }}.{{ item }}" + +- name: make sure the systemd timer is enabled and started + systemd: + daemon_reload: yes + name: "prometheus-node-exporter_{{ textfile_collector_name }}.timer" + state: started + enabled: yes + + +## TODO: install deps for textfile collectors: i.e. smartmontools for collector smartmon diff --git a/roles/monitoring/prometheus/exporter/node/templates/textfile-collector-scripts/apt.service.j2 b/roles/monitoring/prometheus/exporter/node/templates/textfile-collector-scripts/apt.service.j2 new file mode 100644 index 00000000..7eca94fb --- /dev/null +++ b/roles/monitoring/prometheus/exporter/node/templates/textfile-collector-scripts/apt.service.j2 @@ -0,0 +1,31 @@ +[Unit] +Description=Promethues node exporter textfile collector apt + +[Service] +Type=oneshot +Environment=TMPDIR=/var/lib/prometheus-node-exporter/textfile-collector +ExecStart=bash -c "/usr/local/share/prometheus-node-exporter/apt | sponge /var/lib/prometheus-node-exporter/textfile-collector/apt.prom" + +# systemd hardening-options +AmbientCapabilities= +CapabilityBoundingSet= +DeviceAllow=/dev/null rw +DevicePolicy=strict +LockPersonality=true +MemoryDenyWriteExecute=true +NoNewPrivileges=true +PrivateDevices=true +PrivateTmp=true +ProtectControlGroups=true +ProtectHome=true +ProtectKernelModules=true +ProtectKernelTunables=true +ProtectSystem=strict +ReadWritePaths=/var/lib/prometheus-node-exporter/textfile-collector +RemoveIPC=true +RestrictNamespaces=true +RestrictRealtime=true +SystemCallArchitectures=native + +[Install] +WantedBy=multi-user.target diff --git a/roles/monitoring/prometheus/exporter/node/templates/textfile-collector-scripts/apt.timer.j2 b/roles/monitoring/prometheus/exporter/node/templates/textfile-collector-scripts/apt.timer.j2 new file mode 100644 index 00000000..dc473749 --- /dev/null +++ b/roles/monitoring/prometheus/exporter/node/templates/textfile-collector-scripts/apt.timer.j2 @@ -0,0 +1,9 @@ +[Unit] +Description=Promethues node exporter textfile collector apt + +[Timer] +OnBootSec=10s +OnUnitActiveSec=15min + +[Install] +WantedBy=timers.target diff --git a/roles/monitoring/prometheus/exporter/node/templates/textfile-collector-scripts/deleted-libraries.service.j2 b/roles/monitoring/prometheus/exporter/node/templates/textfile-collector-scripts/deleted-libraries.service.j2 new file mode 100644 index 00000000..7b15e558 --- /dev/null +++ b/roles/monitoring/prometheus/exporter/node/templates/textfile-collector-scripts/deleted-libraries.service.j2 @@ -0,0 +1,31 @@ +[Unit] +Description=Promethues node exporter textfile collector deleted-libraries + +[Service] +Type=oneshot +Environment=TMPDIR=/var/lib/prometheus-node-exporter/textfile-collector +ExecStart=bash -c "/usr/local/share/prometheus-node-exporter/deleted-libraries | sponge /var/lib/prometheus-node-exporter/textfile-collector/deleted-libraries.prom" + +# systemd hardening-options +AmbientCapabilities=CAP_SYS_PTRACE +CapabilityBoundingSet=CAP_SYS_PTRACE +DeviceAllow=/dev/null rw +DevicePolicy=strict +LockPersonality=true +MemoryDenyWriteExecute=true +NoNewPrivileges=true +PrivateDevices=true +PrivateTmp=true +ProtectControlGroups=true +ProtectHome=true +ProtectKernelModules=true +ProtectKernelTunables=true +ProtectSystem=strict +ReadWritePaths=/var/lib/prometheus-node-exporter/textfile-collector +RemoveIPC=true +RestrictNamespaces=true +RestrictRealtime=true +SystemCallArchitectures=native + +[Install] +WantedBy=multi-user.target diff --git a/roles/monitoring/prometheus/exporter/node/templates/textfile-collector-scripts/deleted-libraries.timer.j2 b/roles/monitoring/prometheus/exporter/node/templates/textfile-collector-scripts/deleted-libraries.timer.j2 new file mode 100644 index 00000000..c09acecf --- /dev/null +++ b/roles/monitoring/prometheus/exporter/node/templates/textfile-collector-scripts/deleted-libraries.timer.j2 @@ -0,0 +1,9 @@ +[Unit] +Description=Promethues node exporter textfile collector deleted-libraries + +[Timer] +OnBootSec=20s +OnUnitActiveSec=15min + +[Install] +WantedBy=timers.target diff --git a/roles/monitoring/prometheus/exporter/node/templates/textfile-collector-scripts/smartmon.service.j2 b/roles/monitoring/prometheus/exporter/node/templates/textfile-collector-scripts/smartmon.service.j2 new file mode 100644 index 00000000..fc7c9f3f --- /dev/null +++ b/roles/monitoring/prometheus/exporter/node/templates/textfile-collector-scripts/smartmon.service.j2 @@ -0,0 +1,29 @@ +[Unit] +Description=Promethues node exporter textfile collector smartmon + +[Service] +Type=oneshot +Environment=TMPDIR=/var/lib/prometheus-node-exporter/textfile-collector +Environment=LC_NUMERIC=C +ExecStart=bash -c "/usr/local/share/prometheus-node-exporter/smartmon | sponge /var/lib/prometheus-node-exporter/textfile-collector/smartmon.prom" + +# systemd hardening-options +AmbientCapabilities= +CapabilityBoundingSet= +LockPersonality=true +MemoryDenyWriteExecute=true +NoNewPrivileges=true +PrivateTmp=true +ProtectControlGroups=true +ProtectHome=true +ProtectKernelModules=true +ProtectKernelTunables=true +ProtectSystem=strict +ReadWritePaths=/var/lib/prometheus-node-exporter/textfile-collector +RemoveIPC=true +RestrictNamespaces=true +RestrictRealtime=true +SystemCallArchitectures=native + +[Install] +WantedBy=multi-user.target diff --git a/roles/monitoring/prometheus/exporter/node/templates/textfile-collector-scripts/smartmon.timer.j2 b/roles/monitoring/prometheus/exporter/node/templates/textfile-collector-scripts/smartmon.timer.j2 new file mode 100644 index 00000000..576f5a9f --- /dev/null +++ b/roles/monitoring/prometheus/exporter/node/templates/textfile-collector-scripts/smartmon.timer.j2 @@ -0,0 +1,9 @@ +[Unit] +Description=Promethues node exporter textfile collector smartmon + +[Timer] +OnBootSec=30s +OnUnitActiveSec=15min + +[Install] +WantedBy=timers.target diff --git a/roles/monitoring/prometheus/exporter/nut/handlers/main.yml b/roles/monitoring/prometheus/exporter/nut/handlers/main.yml index 6e10f43b..edd87ed5 100644 --- a/roles/monitoring/prometheus/exporter/nut/handlers/main.yml +++ b/roles/monitoring/prometheus/exporter/nut/handlers/main.yml @@ -4,8 +4,7 @@ name: prometheus-nut-exporter state: restarted -- name: reload prometheus-exporter-exporter +- name: reload ngnix service: - name: prometheus-exporter-exporter - ## TODO: implement reload once exporter_exporter supports this... - state: restarted + name: nginx + state: reloaded diff --git a/roles/monitoring/prometheus/exporter/nut/tasks/main.yml b/roles/monitoring/prometheus/exporter/nut/tasks/main.yml index 519ac7a0..f602472d 100644 --- a/roles/monitoring/prometheus/exporter/nut/tasks/main.yml +++ b/roles/monitoring/prometheus/exporter/nut/tasks/main.yml @@ -1,4 +1,5 @@ --- + ## TODO: pin version - name: install apt packages apt: name: prom-exporter-nut @@ -20,9 +21,11 @@ - name: register exporter copy: content: | - method: http - http: - port: 9199 - path: /ups_metrics - dest: /etc/prometheus/exporter/enabled/nut.yml - notify: reload prometheus-exporter-exporter + location = /nut { + proxy_pass http://127.0.0.1:9199/metrics; + } + location = /nut/ups { + proxy_pass http://127.0.0.1:9199/ups_metrics; + } + dest: /etc/prometheus/exporter/nut.locations + notify: reload nginx |