add some basic prometheus node exporter textfile collector scripts

9 files changed, 253 insertions, 1 deletions

diff --git a/roles/monitoring/prometheus/exporter/node/defaults/main.yml b/roles/monitoring/prometheus/exporter/node/defaults/main.yml
index 56227fbb..4a9b40cd 100644
--- a/roles/monitoring/prometheus/exporter/node/defaults/main.yml
+++ b/roles/monitoring/prometheus/exporter/node/defaults/main.yml
@@ -9,4 +9,7 @@ prometheus_exporter_node_timesync_collector: "{{ _prometheus_exporter_node_time_
 
 prometheus_exporter_node_disable_collectors: []
 prometheus_exporter_node_extra_collectors:
-- "{{ prometheus_exporter_node_timesync_collector }}"
+  - "{{ prometheus_exporter_node_timesync_collector }}"
+
+prometheus_exporter_node_textfile_collector_scripts:
+  - deleted-libraries
diff --git a/roles/monitoring/prometheus/exporter/node/files/apt b/roles/monitoring/prometheus/exporter/node/files/apt
new file mode 100755
index 00000000..015addb0
--- /dev/null
+++ b/roles/monitoring/prometheus/exporter/node/files/apt
@@ -0,0 +1,40 @@
+#!/bin/bash
+#
+# Description: Expose metrics from apt updates.
+#
+# Author: Ben Kochie <superq@gmail.com>
+
+upgrades="$(/usr/bin/apt-get --just-print dist-upgrade \
+  | /usr/bin/awk -F'[()]' \
+      '/^Inst/ { sub("^[^ ]+ ", "", $2); gsub(" ","",$2);
+                 sub("\\[", " ", $2); sub("\\]", "", $2); print $2 }' \
+  | /usr/bin/sort \
+  | /usr/bin/uniq -c \
+  | awk '{ gsub(/\\\\/, "\\\\", $2); gsub(/"/, "\\\"", $2);
+           gsub(/\[/, "", $3); gsub(/\]/, "", $3);
+           print "apt_upgrades_pending{origin=\"" $2 "\",arch=\"" $NF "\"} " $1}'
+)"
+
+autoremove="$(/usr/bin/apt-get --just-print autoremove \
+  | /usr/bin/awk '/^Remv/{a++}END{printf "apt_autoremove_pending %d", a}'
+)"
+
+echo '# HELP apt_upgrades_pending Apt package pending updates by origin.'
+echo '# TYPE apt_upgrades_pending gauge'
+if [[ -n "${upgrades}" ]] ; then
+  echo "${upgrades}"
+else
+  echo 'apt_upgrades_pending{origin="",arch=""} 0'
+fi
+
+echo '# HELP apt_autoremove_pending Apt package pending autoremove.'
+echo '# TYPE apt_autoremove_pending gauge'
+echo "${autoremove}"
+
+echo '# HELP node_reboot_required Node reboot is required for software updates.'
+echo '# TYPE node_reboot_required gauge'
+if [[ -f '/run/reboot-required' ]] ; then
+  echo 'node_reboot_required 1'
+else
+  echo 'node_reboot_required 0'
+fi
diff --git a/roles/monitoring/prometheus/exporter/node/files/deleted-libraries b/roles/monitoring/prometheus/exporter/node/files/deleted-libraries
new file mode 100755
index 00000000..e3e19cbd
--- /dev/null
+++ b/roles/monitoring/prometheus/exporter/node/files/deleted-libraries
@@ -0,0 +1,75 @@
+#!/usr/bin/env python3
+"""
+Script to count the number of deleted libraries that are linked by running
+processes and expose a summary as Prometheus metrics.
+
+The aim is to discover processes that are still using libraries that have since
+been updated, perhaps due security vulnerabilities.
+"""
+
+import errno
+import glob
+import os
+import sys
+
+
+def main():
+    processes_linking_deleted_libraries = {}
+
+    for path in glob.glob('/proc/*/maps'):
+        try:
+            with open(path, 'rb') as file:
+                for line in file:
+                    part = line.decode().strip().split()
+
+                    if len(part) == 7:
+                        library = part[5]
+                        comment = part[6]
+
+                        if '/lib/' in library and '(deleted)' in comment:
+                            if path not in processes_linking_deleted_libraries:
+                                processes_linking_deleted_libraries[path] = {}
+
+                                if library in processes_linking_deleted_libraries[path]:
+                                    processes_linking_deleted_libraries[path][library] += 1
+                                else:
+                                    processes_linking_deleted_libraries[path][library] = 1
+        except EnvironmentError as e:
+            # Ignore non-existent files, since the files may have changed since
+            # we globbed.
+            if e.errno != errno.ENOENT:
+                sys.exit('Failed to open file: {0}'.format(path))
+
+    num_processes_per_library = {}
+
+    for process, library_count in processes_linking_deleted_libraries.items():
+        libraries_seen = set()
+        for library, count in library_count.items():
+            if library in libraries_seen:
+                continue
+
+            libraries_seen.add(library)
+            if library in num_processes_per_library:
+                num_processes_per_library[library] += 1
+            else:
+                num_processes_per_library[library] = 1
+
+    metric_name = 'node_processes_linking_deleted_libraries'
+    description = 'Count of running processes that link a deleted library'
+    print('# HELP {0} {1}'.format(metric_name, description))
+    print('# TYPE {0} gauge'.format(metric_name))
+
+    for library, count in num_processes_per_library.items():
+        dir_path, basename = os.path.split(library)
+        basename = basename.replace('"', '\\"')
+        dir_path = dir_path.replace('"', '\\"')
+        print('{0}{{library_path="{1}", library_name="{2}"}} {3}'.format(
+            metric_name,
+            dir_path,
+            basename,
+            count)
+        )
+
+
+if __name__ == "__main__":
+    main()
diff --git a/roles/monitoring/prometheus/exporter/node/tasks/main.yml b/roles/monitoring/prometheus/exporter/node/tasks/main.yml
index 3fa0a1ec..61e385f7 100644
--- a/roles/monitoring/prometheus/exporter/node/tasks/main.yml
+++ b/roles/monitoring/prometheus/exporter/node/tasks/main.yml
@@ -31,3 +31,38 @@
         port: 9100
     dest: /etc/prometheus/exporter/exporter/node.yml
   notify: reload prometheus-exporter-exporter
+
+- name: create directory for textfile collector scripts
+  file:
+    path: /usr/local/lib/prometheus-node-exporter
+    state: directory
+
+- name: install textfile collector script wrapper
+  copy:
+    content: |
+      #!/bin/bash
+
+      if [ -z "$1" ]; then
+        echo "Please specify which collector script to call!"
+        exit 1
+      fi
+      collector="$1"
+
+      set -e
+      rm -f "/var/lib/prometheus-node-exporter/textfile-collector/$collector.prom".*
+      "/usr/local/lib/prometheus-node-exporter/$collector" > "/var/lib/prometheus-node-exporter/textfile-collector/$collector.prom.$$"
+      mv "/var/lib/prometheus-node-exporter/textfile-collector/$collector.prom.$$" "/var/lib/prometheus-node-exporter/textfile-collector/$collector.prom"
+    dest: /usr/local/lib/prometheus-node-exporter/run-collector
+    mode: 0755
+
+- name: install the apt textfile collector script
+  when: ansible_pkg_mgr == "apt"
+  vars:
+    textfile_collector_name: "apt"
+  include_tasks: textfile_collector_script.yml
+
+- name: install all other textfile collector scripts
+  loop: "{{ prometheus_exporter_node_textfile_collector_scripts }}"
+  loop_control:
+    loop_var: textfile_collector_name
+  include_tasks: textfile_collector_script.yml
diff --git a/roles/monitoring/prometheus/exporter/node/tasks/textfile_collector_script.yml b/roles/monitoring/prometheus/exporter/node/tasks/textfile_collector_script.yml
new file mode 100644
index 00000000..1a39bb4c
--- /dev/null
+++ b/roles/monitoring/prometheus/exporter/node/tasks/textfile_collector_script.yml
@@ -0,0 +1,21 @@
+---
+- name: install the collector script
+  copy:
+    src: "{{ textfile_collector_name }}"
+    dest: "/usr/local/lib/prometheus-node-exporter/{{ textfile_collector_name }}"
+    mode: 0755
+
+- name: install systemd service units
+  loop:
+  - service
+  - timer
+  template:
+    src: "textfile-collector-scripts/{{ textfile_collector_name }}.{{ item }}.j2"
+    dest: "/etc/systemd/system/prometheus-node-exporter_{{ textfile_collector_name }}.{{ item }}"
+
+- name: make sure the systemd timer is enabled and started
+  systemd:
+    daemon_reload: yes
+    name: "prometheus-node-exporter_{{ textfile_collector_name }}.timer"
+    state: started
+    enabled: yes
diff --git a/roles/monitoring/prometheus/exporter/node/templates/textfile-collector-scripts/apt.service.j2 b/roles/monitoring/prometheus/exporter/node/templates/textfile-collector-scripts/apt.service.j2
new file mode 100644
index 00000000..b0e9d167
--- /dev/null
+++ b/roles/monitoring/prometheus/exporter/node/templates/textfile-collector-scripts/apt.service.j2
@@ -0,0 +1,30 @@
+[Unit]
+Description=Promethues node exporter textfile collector apt
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/lib/prometheus-node-exporter/run-collector apt
+
+# systemd hardening-options
+AmbientCapabilities=
+CapabilityBoundingSet=
+DeviceAllow=/dev/null rw
+DevicePolicy=strict
+LockPersonality=true
+MemoryDenyWriteExecute=true
+NoNewPrivileges=true
+PrivateDevices=true
+PrivateTmp=true
+ProtectControlGroups=true
+ProtectHome=true
+ProtectKernelModules=true
+ProtectKernelTunables=true
+ProtectSystem=strict
+ReadWritePaths=/var/lib/prometheus-node-exporter/textfile-collector
+RemoveIPC=true
+RestrictNamespaces=true
+RestrictRealtime=true
+SystemCallArchitectures=native
+
+[Install]
+WantedBy=multi-user.target
diff --git a/roles/monitoring/prometheus/exporter/node/templates/textfile-collector-scripts/apt.timer.j2 b/roles/monitoring/prometheus/exporter/node/templates/textfile-collector-scripts/apt.timer.j2
new file mode 100644
index 00000000..5e7d3062
--- /dev/null
+++ b/roles/monitoring/prometheus/exporter/node/templates/textfile-collector-scripts/apt.timer.j2
@@ -0,0 +1,9 @@
+[Unit]
+Description=Promethues node exporter textfile collector apt
+
+[Timer]
+OnCalendar=*-*-* *:1/30:17
+AccuracySec=10s
+
+[Install]
+WantedBy=timers.target
diff --git a/roles/monitoring/prometheus/exporter/node/templates/textfile-collector-scripts/deleted-libraries.service.j2 b/roles/monitoring/prometheus/exporter/node/templates/textfile-collector-scripts/deleted-libraries.service.j2
new file mode 100644
index 00000000..9dbc822f
--- /dev/null
+++ b/roles/monitoring/prometheus/exporter/node/templates/textfile-collector-scripts/deleted-libraries.service.j2
@@ -0,0 +1,30 @@
+[Unit]
+Description=Promethues node exporter textfile collector deleted-libraries
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/lib/prometheus-node-exporter/run-collector deleted-libraries
+
+# systemd hardening-options
+AmbientCapabilities=CAP_SYS_PTRACE
+CapabilityBoundingSet=CAP_SYS_PTRACE
+DeviceAllow=/dev/null rw
+DevicePolicy=strict
+LockPersonality=true
+MemoryDenyWriteExecute=true
+NoNewPrivileges=true
+PrivateDevices=true
+PrivateTmp=true
+ProtectControlGroups=true
+ProtectHome=true
+ProtectKernelModules=true
+ProtectKernelTunables=true
+ProtectSystem=strict
+ReadWritePaths=/var/lib/prometheus-node-exporter/textfile-collector
+RemoveIPC=true
+RestrictNamespaces=true
+RestrictRealtime=true
+SystemCallArchitectures=native
+
+[Install]
+WantedBy=multi-user.target
diff --git a/roles/monitoring/prometheus/exporter/node/templates/textfile-collector-scripts/deleted-libraries.timer.j2 b/roles/monitoring/prometheus/exporter/node/templates/textfile-collector-scripts/deleted-libraries.timer.j2
new file mode 100644
index 00000000..8f38050a
--- /dev/null
+++ b/roles/monitoring/prometheus/exporter/node/templates/textfile-collector-scripts/deleted-libraries.timer.j2
@@ -0,0 +1,9 @@
+[Unit]
+Description=Promethues node exporter textfile collector deleted-libraries
+
+[Timer]
+OnCalendar=*-*-* *:2/30:22
+AccuracySec=10s
+
+[Install]
+WantedBy=timers.target