From 6320da1262c1f44ac773c6b6578a59ba286ce973 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Fri, 24 Sep 2021 00:57:01 +0200 Subject: add some basic prometheus node exporter textfile collector scripts --- .../prometheus/exporter/node/defaults/main.yml | 5 +- .../monitoring/prometheus/exporter/node/files/apt | 40 ++++++++++++ .../exporter/node/files/deleted-libraries | 75 ++++++++++++++++++++++ .../prometheus/exporter/node/tasks/main.yml | 35 ++++++++++ .../node/tasks/textfile_collector_script.yml | 21 ++++++ .../textfile-collector-scripts/apt.service.j2 | 30 +++++++++ .../textfile-collector-scripts/apt.timer.j2 | 9 +++ .../deleted-libraries.service.j2 | 30 +++++++++ .../deleted-libraries.timer.j2 | 9 +++ 9 files changed, 253 insertions(+), 1 deletion(-) create mode 100755 roles/monitoring/prometheus/exporter/node/files/apt create mode 100755 roles/monitoring/prometheus/exporter/node/files/deleted-libraries create mode 100644 roles/monitoring/prometheus/exporter/node/tasks/textfile_collector_script.yml create mode 100644 roles/monitoring/prometheus/exporter/node/templates/textfile-collector-scripts/apt.service.j2 create mode 100644 roles/monitoring/prometheus/exporter/node/templates/textfile-collector-scripts/apt.timer.j2 create mode 100644 roles/monitoring/prometheus/exporter/node/templates/textfile-collector-scripts/deleted-libraries.service.j2 create mode 100644 roles/monitoring/prometheus/exporter/node/templates/textfile-collector-scripts/deleted-libraries.timer.j2 (limited to 'roles/monitoring/prometheus') diff --git a/roles/monitoring/prometheus/exporter/node/defaults/main.yml b/roles/monitoring/prometheus/exporter/node/defaults/main.yml index 56227fbb..4a9b40cd 100644 --- a/roles/monitoring/prometheus/exporter/node/defaults/main.yml +++ b/roles/monitoring/prometheus/exporter/node/defaults/main.yml @@ -9,4 +9,7 @@ prometheus_exporter_node_timesync_collector: "{{ _prometheus_exporter_node_time_ prometheus_exporter_node_disable_collectors: [] prometheus_exporter_node_extra_collectors: -- "{{ prometheus_exporter_node_timesync_collector }}" + - "{{ prometheus_exporter_node_timesync_collector }}" + +prometheus_exporter_node_textfile_collector_scripts: + - deleted-libraries diff --git a/roles/monitoring/prometheus/exporter/node/files/apt b/roles/monitoring/prometheus/exporter/node/files/apt new file mode 100755 index 00000000..015addb0 --- /dev/null +++ b/roles/monitoring/prometheus/exporter/node/files/apt @@ -0,0 +1,40 @@ +#!/bin/bash +# +# Description: Expose metrics from apt updates. +# +# Author: Ben Kochie + +upgrades="$(/usr/bin/apt-get --just-print dist-upgrade \ + | /usr/bin/awk -F'[()]' \ + '/^Inst/ { sub("^[^ ]+ ", "", $2); gsub(" ","",$2); + sub("\\[", " ", $2); sub("\\]", "", $2); print $2 }' \ + | /usr/bin/sort \ + | /usr/bin/uniq -c \ + | awk '{ gsub(/\\\\/, "\\\\", $2); gsub(/"/, "\\\"", $2); + gsub(/\[/, "", $3); gsub(/\]/, "", $3); + print "apt_upgrades_pending{origin=\"" $2 "\",arch=\"" $NF "\"} " $1}' +)" + +autoremove="$(/usr/bin/apt-get --just-print autoremove \ + | /usr/bin/awk '/^Remv/{a++}END{printf "apt_autoremove_pending %d", a}' +)" + +echo '# HELP apt_upgrades_pending Apt package pending updates by origin.' +echo '# TYPE apt_upgrades_pending gauge' +if [[ -n "${upgrades}" ]] ; then + echo "${upgrades}" +else + echo 'apt_upgrades_pending{origin="",arch=""} 0' +fi + +echo '# HELP apt_autoremove_pending Apt package pending autoremove.' +echo '# TYPE apt_autoremove_pending gauge' +echo "${autoremove}" + +echo '# HELP node_reboot_required Node reboot is required for software updates.' +echo '# TYPE node_reboot_required gauge' +if [[ -f '/run/reboot-required' ]] ; then + echo 'node_reboot_required 1' +else + echo 'node_reboot_required 0' +fi diff --git a/roles/monitoring/prometheus/exporter/node/files/deleted-libraries b/roles/monitoring/prometheus/exporter/node/files/deleted-libraries new file mode 100755 index 00000000..e3e19cbd --- /dev/null +++ b/roles/monitoring/prometheus/exporter/node/files/deleted-libraries @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +""" +Script to count the number of deleted libraries that are linked by running +processes and expose a summary as Prometheus metrics. + +The aim is to discover processes that are still using libraries that have since +been updated, perhaps due security vulnerabilities. +""" + +import errno +import glob +import os +import sys + + +def main(): + processes_linking_deleted_libraries = {} + + for path in glob.glob('/proc/*/maps'): + try: + with open(path, 'rb') as file: + for line in file: + part = line.decode().strip().split() + + if len(part) == 7: + library = part[5] + comment = part[6] + + if '/lib/' in library and '(deleted)' in comment: + if path not in processes_linking_deleted_libraries: + processes_linking_deleted_libraries[path] = {} + + if library in processes_linking_deleted_libraries[path]: + processes_linking_deleted_libraries[path][library] += 1 + else: + processes_linking_deleted_libraries[path][library] = 1 + except EnvironmentError as e: + # Ignore non-existent files, since the files may have changed since + # we globbed. + if e.errno != errno.ENOENT: + sys.exit('Failed to open file: {0}'.format(path)) + + num_processes_per_library = {} + + for process, library_count in processes_linking_deleted_libraries.items(): + libraries_seen = set() + for library, count in library_count.items(): + if library in libraries_seen: + continue + + libraries_seen.add(library) + if library in num_processes_per_library: + num_processes_per_library[library] += 1 + else: + num_processes_per_library[library] = 1 + + metric_name = 'node_processes_linking_deleted_libraries' + description = 'Count of running processes that link a deleted library' + print('# HELP {0} {1}'.format(metric_name, description)) + print('# TYPE {0} gauge'.format(metric_name)) + + for library, count in num_processes_per_library.items(): + dir_path, basename = os.path.split(library) + basename = basename.replace('"', '\\"') + dir_path = dir_path.replace('"', '\\"') + print('{0}{{library_path="{1}", library_name="{2}"}} {3}'.format( + metric_name, + dir_path, + basename, + count) + ) + + +if __name__ == "__main__": + main() diff --git a/roles/monitoring/prometheus/exporter/node/tasks/main.yml b/roles/monitoring/prometheus/exporter/node/tasks/main.yml index 3fa0a1ec..61e385f7 100644 --- a/roles/monitoring/prometheus/exporter/node/tasks/main.yml +++ b/roles/monitoring/prometheus/exporter/node/tasks/main.yml @@ -31,3 +31,38 @@ port: 9100 dest: /etc/prometheus/exporter/exporter/node.yml notify: reload prometheus-exporter-exporter + +- name: create directory for textfile collector scripts + file: + path: /usr/local/lib/prometheus-node-exporter + state: directory + +- name: install textfile collector script wrapper + copy: + content: | + #!/bin/bash + + if [ -z "$1" ]; then + echo "Please specify which collector script to call!" + exit 1 + fi + collector="$1" + + set -e + rm -f "/var/lib/prometheus-node-exporter/textfile-collector/$collector.prom".* + "/usr/local/lib/prometheus-node-exporter/$collector" > "/var/lib/prometheus-node-exporter/textfile-collector/$collector.prom.$$" + mv "/var/lib/prometheus-node-exporter/textfile-collector/$collector.prom.$$" "/var/lib/prometheus-node-exporter/textfile-collector/$collector.prom" + dest: /usr/local/lib/prometheus-node-exporter/run-collector + mode: 0755 + +- name: install the apt textfile collector script + when: ansible_pkg_mgr == "apt" + vars: + textfile_collector_name: "apt" + include_tasks: textfile_collector_script.yml + +- name: install all other textfile collector scripts + loop: "{{ prometheus_exporter_node_textfile_collector_scripts }}" + loop_control: + loop_var: textfile_collector_name + include_tasks: textfile_collector_script.yml diff --git a/roles/monitoring/prometheus/exporter/node/tasks/textfile_collector_script.yml b/roles/monitoring/prometheus/exporter/node/tasks/textfile_collector_script.yml new file mode 100644 index 00000000..1a39bb4c --- /dev/null +++ b/roles/monitoring/prometheus/exporter/node/tasks/textfile_collector_script.yml @@ -0,0 +1,21 @@ +--- +- name: install the collector script + copy: + src: "{{ textfile_collector_name }}" + dest: "/usr/local/lib/prometheus-node-exporter/{{ textfile_collector_name }}" + mode: 0755 + +- name: install systemd service units + loop: + - service + - timer + template: + src: "textfile-collector-scripts/{{ textfile_collector_name }}.{{ item }}.j2" + dest: "/etc/systemd/system/prometheus-node-exporter_{{ textfile_collector_name }}.{{ item }}" + +- name: make sure the systemd timer is enabled and started + systemd: + daemon_reload: yes + name: "prometheus-node-exporter_{{ textfile_collector_name }}.timer" + state: started + enabled: yes diff --git a/roles/monitoring/prometheus/exporter/node/templates/textfile-collector-scripts/apt.service.j2 b/roles/monitoring/prometheus/exporter/node/templates/textfile-collector-scripts/apt.service.j2 new file mode 100644 index 00000000..b0e9d167 --- /dev/null +++ b/roles/monitoring/prometheus/exporter/node/templates/textfile-collector-scripts/apt.service.j2 @@ -0,0 +1,30 @@ +[Unit] +Description=Promethues node exporter textfile collector apt + +[Service] +Type=oneshot +ExecStart=/usr/local/lib/prometheus-node-exporter/run-collector apt + +# systemd hardening-options +AmbientCapabilities= +CapabilityBoundingSet= +DeviceAllow=/dev/null rw +DevicePolicy=strict +LockPersonality=true +MemoryDenyWriteExecute=true +NoNewPrivileges=true +PrivateDevices=true +PrivateTmp=true +ProtectControlGroups=true +ProtectHome=true +ProtectKernelModules=true +ProtectKernelTunables=true +ProtectSystem=strict +ReadWritePaths=/var/lib/prometheus-node-exporter/textfile-collector +RemoveIPC=true +RestrictNamespaces=true +RestrictRealtime=true +SystemCallArchitectures=native + +[Install] +WantedBy=multi-user.target diff --git a/roles/monitoring/prometheus/exporter/node/templates/textfile-collector-scripts/apt.timer.j2 b/roles/monitoring/prometheus/exporter/node/templates/textfile-collector-scripts/apt.timer.j2 new file mode 100644 index 00000000..5e7d3062 --- /dev/null +++ b/roles/monitoring/prometheus/exporter/node/templates/textfile-collector-scripts/apt.timer.j2 @@ -0,0 +1,9 @@ +[Unit] +Description=Promethues node exporter textfile collector apt + +[Timer] +OnCalendar=*-*-* *:1/30:17 +AccuracySec=10s + +[Install] +WantedBy=timers.target diff --git a/roles/monitoring/prometheus/exporter/node/templates/textfile-collector-scripts/deleted-libraries.service.j2 b/roles/monitoring/prometheus/exporter/node/templates/textfile-collector-scripts/deleted-libraries.service.j2 new file mode 100644 index 00000000..9dbc822f --- /dev/null +++ b/roles/monitoring/prometheus/exporter/node/templates/textfile-collector-scripts/deleted-libraries.service.j2 @@ -0,0 +1,30 @@ +[Unit] +Description=Promethues node exporter textfile collector deleted-libraries + +[Service] +Type=oneshot +ExecStart=/usr/local/lib/prometheus-node-exporter/run-collector deleted-libraries + +# systemd hardening-options +AmbientCapabilities=CAP_SYS_PTRACE +CapabilityBoundingSet=CAP_SYS_PTRACE +DeviceAllow=/dev/null rw +DevicePolicy=strict +LockPersonality=true +MemoryDenyWriteExecute=true +NoNewPrivileges=true +PrivateDevices=true +PrivateTmp=true +ProtectControlGroups=true +ProtectHome=true +ProtectKernelModules=true +ProtectKernelTunables=true +ProtectSystem=strict +ReadWritePaths=/var/lib/prometheus-node-exporter/textfile-collector +RemoveIPC=true +RestrictNamespaces=true +RestrictRealtime=true +SystemCallArchitectures=native + +[Install] +WantedBy=multi-user.target diff --git a/roles/monitoring/prometheus/exporter/node/templates/textfile-collector-scripts/deleted-libraries.timer.j2 b/roles/monitoring/prometheus/exporter/node/templates/textfile-collector-scripts/deleted-libraries.timer.j2 new file mode 100644 index 00000000..8f38050a --- /dev/null +++ b/roles/monitoring/prometheus/exporter/node/templates/textfile-collector-scripts/deleted-libraries.timer.j2 @@ -0,0 +1,9 @@ +[Unit] +Description=Promethues node exporter textfile collector deleted-libraries + +[Timer] +OnCalendar=*-*-* *:2/30:22 +AccuracySec=10s + +[Install] +WantedBy=timers.target -- cgit v1.2.3