From 695131994b5a749e129fb304e8ba709acd37afe8 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Fri, 26 May 2023 21:34:14 +0200 Subject: add support for chrony_exporter (replaces textfile collector) --- roles/monitoring/grafana/defaults/main.yml | 2 +- .../monitoring/grafana/files/dashboard-chrony.json | 1210 ++++++++++++++++++++ .../grafana/files/dashboard-chronyd.json | 1146 ------------------ .../prometheus/exporter/chrony/defaults/main.yml | 6 + .../prometheus/exporter/chrony/handlers/main.yml | 15 + .../prometheus/exporter/chrony/tasks/main.yml | 65 ++ .../exporter/chrony/templates/service.j2 | 31 + roles/monitoring/prometheus/exporter/meta/main.yml | 2 + .../prometheus/exporter/node/defaults/main.yml | 1 - .../templates/textfile-collector-scripts/chrony.j2 | 138 --- .../textfile-collector-scripts/chrony.service.j2 | 33 - .../textfile-collector-scripts/chrony.timer.j2 | 9 - .../prometheus/server/defaults/main/main.yml | 1 + .../server/defaults/main/rules_chrony.yml | 14 + 14 files changed, 1345 insertions(+), 1328 deletions(-) create mode 100644 roles/monitoring/grafana/files/dashboard-chrony.json delete mode 100644 roles/monitoring/grafana/files/dashboard-chronyd.json create mode 100644 roles/monitoring/prometheus/exporter/chrony/defaults/main.yml create mode 100644 roles/monitoring/prometheus/exporter/chrony/handlers/main.yml create mode 100644 roles/monitoring/prometheus/exporter/chrony/tasks/main.yml create mode 100644 roles/monitoring/prometheus/exporter/chrony/templates/service.j2 delete mode 100644 roles/monitoring/prometheus/exporter/node/templates/textfile-collector-scripts/chrony.j2 delete mode 100644 roles/monitoring/prometheus/exporter/node/templates/textfile-collector-scripts/chrony.service.j2 delete mode 100644 roles/monitoring/prometheus/exporter/node/templates/textfile-collector-scripts/chrony.timer.j2 create mode 100644 roles/monitoring/prometheus/server/defaults/main/rules_chrony.yml (limited to 'roles/monitoring') diff --git a/roles/monitoring/grafana/defaults/main.yml b/roles/monitoring/grafana/defaults/main.yml index 10eac947..8a113e2d 100644 --- a/roles/monitoring/grafana/defaults/main.yml +++ b/roles/monitoring/grafana/defaults/main.yml @@ -41,7 +41,7 @@ grafana_datasources: [] grafana_dashboards: [] # - file: node-full # datasource: "Prometheus" -# - file: chronyd +# - file: chrony # datasource: "Prometheus" # - file: environment-sensors # datasource: "Prometheus" diff --git a/roles/monitoring/grafana/files/dashboard-chrony.json b/roles/monitoring/grafana/files/dashboard-chrony.json new file mode 100644 index 00000000..d6ae5c4f --- /dev/null +++ b/roles/monitoring/grafana/files/dashboard-chrony.json @@ -0,0 +1,1210 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "10.0.2" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 11, + "panels": [], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "refId": "A" + } + ], + "title": "Tracking", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 4, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "name" + }, + "pluginVersion": "10.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "chrony_tracking_info{instance=\"$instance\"}", + "format": "time_series", + "instant": true, + "interval": "", + "legendFormat": "{{ tracking_address }}", + "refId": "A" + } + ], + "title": "Reference Source Address", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 4 + }, + { + "color": "red", + "value": 10 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "value" + }, + "pluginVersion": "10.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "chrony_tracking_stratum{instance=\"$instance\"}", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Stratum", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "chrony_tracking_last_offset_seconds{instance=\"$instance\"}", + "interval": "", + "legendFormat": "Last Offset", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "chrony_tracking_rms_offset_seconds{instance=\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "RMS Offset (long term average)", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "chrony_tracking_system_time_seconds{instance=\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "System Time", + "range": true, + "refId": "C" + } + ], + "title": "Offset", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 0, + "y": 4 + }, + "id": 2, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "name" + }, + "pluginVersion": "10.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "chrony_tracking_info{instance=\"$instance\"}", + "instant": true, + "interval": "", + "legendFormat": "{{ tracking_refid }}", + "refId": "A" + } + ], + "title": "Reference Source ID", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Absolute bound on the computer’s clock accuracy (assuming the stratum-1 computer is correct)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "instance:chrony_clock_error_seconds:abs{instance=\"$instance\"}", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Clock Error", + "range": true, + "refId": "A" + } + ], + "title": "Maximum Clock Error", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "chrony_tracking_root_delay_seconds{instance=\"$instance\"}", + "interval": "", + "legendFormat": "Root delay", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "chrony_tracking_root_dispersion_seconds{instance=\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "Root dispersion", + "range": true, + "refId": "B" + } + ], + "title": "Source delay", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 15 + }, + "id": 13, + "panels": [], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "refId": "A" + } + ], + "title": "Sources", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "mode": "basic", + "type": "color-background" + }, + "filterable": false, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "State" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "options": { + "candidate": { + "color": "light-green", + "index": 4, + "text": "combined" + }, + "falseticker": { + "color": "orange", + "index": 2, + "text": "falseticker" + }, + "jittery": { + "color": "yellow", + "index": 3, + "text": "jittery" + }, + "outlier": { + "color": "transparent", + "index": 5, + "text": "not combined" + }, + "sync": { + "color": "green", + "index": 0, + "text": "syncronized" + }, + "unreach": { + "color": "red", + "index": 1, + "text": "unreachable" + } + }, + "type": "value" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Polling Interval" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 19, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "frameIndex": 0, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "10.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "exemplar": false, + "expr": "chrony_sources_state_info{instance=\"$instance\"}", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "__auto", + "range": false, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "exemplar": false, + "expr": "chrony_sources_stratum{instance=\"$instance\"}", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "exemplar": false, + "expr": "chrony_sources_polling_interval_seconds{instance=\"$instance\"}", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "__auto", + "range": false, + "refId": "C" + } + ], + "title": "Current Status", + "transformations": [ + { + "id": "joinByField", + "options": { + "byField": "source_address", + "mode": "outer" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Time 2": true, + "Value #A": true, + "Value #B": false, + "__name__": true, + "__name__ 2": true, + "instance": true, + "instance 2": true, + "job": true, + "job 2": true, + "source_mode": true, + "source_name 2": true, + "source_name 3": true + }, + "indexByName": { + "Time 1": 1, + "Time 2": 9, + "Time 3": 15, + "Value #A": 8, + "Value #B": 14, + "Value #C": 20, + "__name__ 1": 2, + "__name__ 2": 10, + "__name__ 3": 16, + "instance 1": 3, + "instance 2": 11, + "instance 3": 17, + "job 1": 4, + "job 2": 12, + "job 3": 18, + "source_address": 0, + "source_mode": 5, + "source_name 1": 6, + "source_name 2": 13, + "source_name 3": 19, + "source_state": 7 + }, + "renameByName": { + "Value #B": "Stratum", + "Value #C": "Polling Interval", + "source_address": "Address", + "source_name": "Name", + "source_name 1": "Name", + "source_state": "State" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 24 + }, + "id": 18, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "exemplar": true, + "expr": "chrony_sources_last_sample_age_seconds{instance=\"$instance\"} < 4294967295", + "interval": "", + "legendFormat": "{{ source_address }} ({{ source_name }})", + "range": true, + "refId": "A" + } + ], + "title": "Last Sample: Age", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 24 + }, + "id": 21, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "chrony_sources_last_sample_offset_seconds{instance=\"$instance\"}", + "interval": "", + "legendFormat": "{{ source_address }} ({{ source_name }})", + "range": true, + "refId": "A" + } + ], + "title": "Last Sample: Offset", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 24 + }, + "id": 20, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "chrony_sources_last_sample_error_margin_seconds{instance=\"$instance\"}", + "interval": "", + "legendFormat": "{{ source_address }} ({{ source_name }})", + "range": true, + "refId": "A" + } + ], + "title": "Last Sample: Error Margin", + "type": "timeseries" + } + ], + "refresh": "30s", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(chrony_up,instance)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "instance", + "options": [], + "query": { + "query": "label_values(chrony_up,instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-12h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "chrony", + "uid": "c252be85-471b-41c9-b1cb-f0e1b8206593", + "version": 3, + "weekStart": "" +} \ No newline at end of file diff --git a/roles/monitoring/grafana/files/dashboard-chronyd.json b/roles/monitoring/grafana/files/dashboard-chronyd.json deleted file mode 100644 index 1a401ffa..00000000 --- a/roles/monitoring/grafana/files/dashboard-chronyd.json +++ /dev/null @@ -1,1146 +0,0 @@ -{ - "__inputs": [ - { - "name": "DS_PROMETHEUS", - "label": "Prometheus", - "description": "", - "type": "datasource", - "pluginId": "prometheus", - "pluginName": "Prometheus" - } - ], - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "8.2.2" - }, - { - "type": "datasource", - "id": "prometheus", - "name": "Prometheus", - "version": "1.0.0" - }, - { - "type": "panel", - "id": "stat", - "name": "Stat", - "version": "" - }, - { - "type": "panel", - "id": "state-timeline", - "name": "State timeline", - "version": "" - }, - { - "type": "panel", - "id": "table", - "name": "Table", - "version": "" - }, - { - "type": "panel", - "id": "timeseries", - "name": "Time series", - "version": "" - } - ], - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "gnetId": null, - "graphTooltip": 1, - "id": null, - "iteration": 1642434896568, - "links": [], - "liveNow": false, - "panels": [ - { - "collapsed": false, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 11, - "panels": [], - "title": "Tracking", - "type": "row" - }, - { - "datasource": null, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 0, - "y": 1 - }, - "id": 4, - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "center", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "name" - }, - "pluginVersion": "8.2.2", - "repeat": null, - "targets": [ - { - "exemplar": false, - "expr": "chronyd_tracking_source{instance=\"$instance\"}", - "format": "time_series", - "instant": true, - "interval": "", - "legendFormat": "{{ value }}", - "refId": "A" - } - ], - "title": "Reference Source", - "type": "stat" - }, - { - "datasource": null, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "#EAB839", - "value": 4 - }, - { - "color": "red", - "value": 10 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 6, - "y": 1 - }, - "id": 3, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "center", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "value" - }, - "pluginVersion": "8.2.2", - "targets": [ - { - "exemplar": false, - "expr": "chronyd_tracking_stratum{instance=\"$instance\"}", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Stratum", - "type": "stat" - }, - { - "datasource": null, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "ppm" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 1 - }, - "id": 6, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ - { - "exemplar": true, - "expr": "chronyd_tracking_frequency_error{instance=\"$instance\"}", - "interval": "", - "legendFormat": "Frequency", - "refId": "A" - }, - { - "exemplar": true, - "expr": "chronyd_tracking_frequency_residual{instance=\"$instance\"}", - "hide": false, - "interval": "", - "legendFormat": "Residual Frequency", - "refId": "B" - }, - { - "exemplar": true, - "expr": "chronyd_tracking_frequency_skew{instance=\"$instance\"}", - "hide": false, - "interval": "", - "legendFormat": "Skew", - "refId": "C" - } - ], - "title": "Frequency", - "type": "timeseries" - }, - { - "datasource": null, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 0, - "y": 4 - }, - "id": 2, - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "center", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "name" - }, - "pluginVersion": "8.2.2", - "targets": [ - { - "exemplar": true, - "expr": "chronyd_tracking_leap_status{instance=\"$instance\"}", - "instant": true, - "interval": "", - "legendFormat": "{{ value }}", - "refId": "A" - } - ], - "title": "Leap second status", - "type": "stat" - }, - { - "datasource": null, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 7 - }, - "id": 8, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ - { - "exemplar": true, - "expr": "chronyd_tracking_last_offset{instance=\"$instance\"}", - "interval": "", - "legendFormat": "Last Offset", - "refId": "A" - }, - { - "exemplar": true, - "expr": "chronyd_tracking_rms_offset{instance=\"$instance\"}", - "hide": false, - "interval": "", - "legendFormat": "RMS Offse (long term average)", - "refId": "B" - }, - { - "exemplar": true, - "expr": "chronyd_tracking_system_time{instance=\"$instance\"}", - "hide": false, - "interval": "", - "legendFormat": "System Time", - "refId": "C" - } - ], - "title": "Offset", - "type": "timeseries" - }, - { - "datasource": null, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 9 - }, - "id": 9, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ - { - "exemplar": true, - "expr": "chronyd_tracking_root_delay{instance=\"$instance\"}", - "interval": "", - "legendFormat": "Root delay", - "refId": "A" - }, - { - "exemplar": true, - "expr": "chronyd_tracking_root_dispersion{instance=\"$instance\"}", - "hide": false, - "interval": "", - "legendFormat": "Root dispersion", - "refId": "B" - } - ], - "title": "Source delay", - "type": "timeseries" - }, - { - "collapsed": false, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 17 - }, - "id": 13, - "panels": [], - "title": "Sources", - "type": "row" - }, - { - "datasource": null, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "displayMode": "color-background-solid", - "filterable": false - }, - "mappings": [ - { - "options": { - "0": { - "index": 0, - "text": "may be in error" - }, - "1": { - "index": 1, - "text": "unusable" - }, - "2": { - "index": 2, - "text": "not combined" - }, - "3": { - "index": 3, - "text": "combined" - }, - "4": { - "index": 4, - "text": "current best" - } - }, - "type": "value" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "transparent", - "value": null - }, - { - "color": "red", - "value": 0 - }, - { - "color": "orange", - "value": 1 - }, - { - "color": "blue", - "value": 2 - }, - { - "color": "light-green", - "value": 3 - }, - { - "color": "green", - "value": 4 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 18 - }, - "id": 19, - "options": { - "frameIndex": 0, - "showHeader": false - }, - "pluginVersion": "8.2.2", - "repeat": null, - "targets": [ - { - "exemplar": false, - "expr": "chronyd_peer_status{instance=\"$instance\"}", - "format": "table", - "instant": true, - "interval": "", - "legendFormat": "{{ remote }}", - "refId": "A" - } - ], - "title": "Current Status", - "transformations": [ - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true, - "__name__": true, - "instance": true, - "job": true, - "mode": true, - "stratum": true - }, - "indexByName": {}, - "renameByName": { - "Time": "" - } - } - }, - { - "id": "sortBy", - "options": { - "fields": {}, - "sort": [ - { - "desc": true, - "field": "Value" - } - ] - } - } - ], - "type": "table" - }, - { - "datasource": null, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "fillOpacity": 74, - "lineWidth": 0 - }, - "mappings": [ - { - "options": { - "0": { - "index": 0, - "text": "unreachable" - }, - "1": { - "index": 1, - "text": "ok" - } - }, - "type": "value" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 18 - }, - "id": 16, - "options": { - "alignValue": "center", - "legend": { - "displayMode": "hidden", - "placement": "bottom" - }, - "mergeValues": true, - "rowHeight": 0.5, - "showValue": "never", - "tooltip": { - "mode": "single" - } - }, - "targets": [ - { - "exemplar": false, - "expr": "max without (stratum) (chronyd_peer_reachable{instance=\"$instance\"})", - "instant": false, - "interval": "", - "legendFormat": "{{ remote }}", - "refId": "A" - } - ], - "title": "Reachability", - "type": "state-timeline" - }, - { - "datasource": null, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 18 - }, - "id": 17, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ - { - "exemplar": true, - "expr": "chronyd_offset_seconds{instance=\"$instance\"}", - "interval": "", - "legendFormat": "{{ remote }}", - "refId": "A" - } - ], - "title": "Offset", - "type": "timeseries" - }, - { - "datasource": null, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ppm" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 25 - }, - "id": 15, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ - { - "exemplar": true, - "expr": "chronyd_freq_ppm{instance=\"$instance\"}", - "interval": "", - "legendFormat": "{{ remote }}", - "refId": "A" - } - ], - "title": "Frequency", - "type": "timeseries" - }, - { - "datasource": null, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ppm" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 25 - }, - "id": 20, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ - { - "exemplar": true, - "expr": "chronyd_freq_skew_ppm{instance=\"$instance\"}", - "interval": "", - "legendFormat": "{{ remote }}", - "refId": "A" - } - ], - "title": "Frequency Skew", - "type": "timeseries" - }, - { - "datasource": null, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 25 - }, - "id": 18, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ - { - "exemplar": true, - "expr": "chronyd_std_dev_seconds{instance=\"$instance\"}", - "interval": "", - "legendFormat": "{{ remote }}", - "refId": "A" - } - ], - "title": "Std. Dev.", - "type": "timeseries" - } - ], - "refresh": "30s", - "schemaVersion": 31, - "style": "dark", - "tags": [], - "templating": { - "list": [ - { - "allValue": null, - "current": {}, - "datasource": "${DS_PROMETHEUS}", - "definition": "label_values(chronyd_tracking_source, instance)", - "description": null, - "error": null, - "hide": 0, - "includeAll": false, - "label": null, - "multi": false, - "name": "instance", - "options": [], - "query": { - "query": "label_values(chronyd_tracking_source, instance)", - "refId": "StandardVariableQuery" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "type": "query" - } - ] - }, - "time": { - "from": "now-12h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "chronyd", - "uid": "eIO_Uyd7k", - "version": 3 -} diff --git a/roles/monitoring/prometheus/exporter/chrony/defaults/main.yml b/roles/monitoring/prometheus/exporter/chrony/defaults/main.yml new file mode 100644 index 00000000..699ed580 --- /dev/null +++ b/roles/monitoring/prometheus/exporter/chrony/defaults/main.yml @@ -0,0 +1,6 @@ +--- +# prometheus_exporter_chrony_version: + +prometheus_exporter_chrony_enable_collectors: + - sources + - tracking diff --git a/roles/monitoring/prometheus/exporter/chrony/handlers/main.yml b/roles/monitoring/prometheus/exporter/chrony/handlers/main.yml new file mode 100644 index 00000000..0c940ca9 --- /dev/null +++ b/roles/monitoring/prometheus/exporter/chrony/handlers/main.yml @@ -0,0 +1,15 @@ +--- +- name: restart prometheus-chrony-exporter + service: + name: prometheus-chrony-exporter + state: restarted + +- name: reload nginx + service: + name: nginx + state: reloaded + +### TODO: remove this once all hosts have been migrated +- name: reload systemd + systemd: + daemon_reload: yes diff --git a/roles/monitoring/prometheus/exporter/chrony/tasks/main.yml b/roles/monitoring/prometheus/exporter/chrony/tasks/main.yml new file mode 100644 index 00000000..f15037ec --- /dev/null +++ b/roles/monitoring/prometheus/exporter/chrony/tasks/main.yml @@ -0,0 +1,65 @@ +--- +- name: generate apt pin file for exporter-chrony package + when: prometheus_exporter_chrony_version is defined + copy: + dest: "/etc/apt/preferences.d/prom-exporter-chrony.pref" + content: | + Package: prom-exporter-chrony + Pin: version {{ prometheus_exporter_chrony_version }}-1 + Pin-Priority: 1001 + +- name: remove apt pin file for exporter-chrony package + when: prometheus_exporter_chrony_version is not defined + file: + path: "/etc/apt/preferences.d/prom-exporter-chrony.pref" + state: absent + +- name: install apt packages + apt: + name: "prom-exporter-chrony{% if prometheus_exporter_chrony_version is defined %}={{ prometheus_exporter_chrony_version }}-1{% endif %}" + state: present + allow_downgrade: yes + notify: restart prometheus-chrony-exporter + +- name: generate systemd service unit + template: + src: service.j2 + dest: /etc/systemd/system/prometheus-chrony-exporter.service + notify: restart prometheus-chrony-exporter + +- name: make sure prometheus-chrony-exporter is enabled and started + systemd: + name: prometheus-chrony-exporter.service + daemon_reload: yes + state: started + enabled: yes + +- name: register exporter + copy: + content: | + location = /chrony { + proxy_pass http://127.0.0.1:9123/metrics; + } + dest: /etc/prometheus/exporter/chrony.locations + notify: reload nginx + + +## TODO: remove these tasks once all hosts have been migrated +- name: make sure the systemd timer for chrony textfile collector is disabled and stopped + systemd: + service: prometheus-node-exporter_chrony.timer + enabled: no + state: stopped + register: result_systemd_stop + failed_when: "result_systemd_stop is failed and 'Could not find the requested service' not in result_systemd_stop.msg" + +- name: remove files from chrony textfile collector + loop: + - /etc/systemd/system/prometheus-node-exporter_chrony.timer + - /etc/systemd/system/prometheus-node-exporter_chrony.service + - /usr/local/share/prometheus-node-exporter/chrony + - /var/lib/prometheus-node-exporter/textfile-collector/chrony.prom + file: + path: "{{ item }}" + state: absent + notify: reload systemd diff --git a/roles/monitoring/prometheus/exporter/chrony/templates/service.j2 b/roles/monitoring/prometheus/exporter/chrony/templates/service.j2 new file mode 100644 index 00000000..cb806649 --- /dev/null +++ b/roles/monitoring/prometheus/exporter/chrony/templates/service.j2 @@ -0,0 +1,31 @@ +[Unit] +Description=Prometheus chrony exporter + +[Service] +Restart=always +User=_chrony +ExecStart=/usr/bin/prometheus-chrony-exporter --web.listen-address="127.0.0.1:9123" --chrony.address=unix:///run/chrony/chronyd.sock {% for collector in prometheus_exporter_chrony_enable_collectors %} --collector.{{ collector }}{% endfor %}{{ '' }} + +# systemd hardening-options +AmbientCapabilities= +CapabilityBoundingSet= +DeviceAllow=/dev/null rw +DevicePolicy=strict +LockPersonality=true +MemoryDenyWriteExecute=true +NoNewPrivileges=true +PrivateDevices=true +PrivateTmp=true +ProtectControlGroups=true +ProtectHome=true +ProtectKernelModules=true +ProtectKernelTunables=true +ProtectSystem=strict +ReadWritePaths=/run/chrony +RemoveIPC=true +RestrictNamespaces=true +RestrictRealtime=true +SystemCallArchitectures=native + +[Install] +WantedBy=multi-user.target diff --git a/roles/monitoring/prometheus/exporter/meta/main.yml b/roles/monitoring/prometheus/exporter/meta/main.yml index 4a427770..10a251f4 100644 --- a/roles/monitoring/prometheus/exporter/meta/main.yml +++ b/roles/monitoring/prometheus/exporter/meta/main.yml @@ -23,4 +23,6 @@ dependencies: when: "'standalone-kubelet' in (prometheus_exporters_default | union(prometheus_exporters_extra))" - role: monitoring/prometheus/exporter/modbus when: "'modbus' in (prometheus_exporters_default | union(prometheus_exporters_extra))" + - role: monitoring/prometheus/exporter/chrony + when: "'chrony' in (prometheus_exporters_default | union(prometheus_exporters_extra))" - role: monitoring/prometheus/exporter/register diff --git a/roles/monitoring/prometheus/exporter/node/defaults/main.yml b/roles/monitoring/prometheus/exporter/node/defaults/main.yml index 3b961a4f..ab4cee38 100644 --- a/roles/monitoring/prometheus/exporter/node/defaults/main.yml +++ b/roles/monitoring/prometheus/exporter/node/defaults/main.yml @@ -18,7 +18,6 @@ prometheus_exporter_node_install_apt_textfile_collector_script: "{{ ansible_pkg_ prometheus_exporter_node_textfile_collector_scripts: - deleted-libraries # - smartmon -# - chrony # - sensors # prometheus_exporter_node_textfile_collector__sensors: diff --git a/roles/monitoring/prometheus/exporter/node/templates/textfile-collector-scripts/chrony.j2 b/roles/monitoring/prometheus/exporter/node/templates/textfile-collector-scripts/chrony.j2 deleted file mode 100644 index 95c6a5d3..00000000 --- a/roles/monitoring/prometheus/exporter/node/templates/textfile-collector-scripts/chrony.j2 +++ /dev/null @@ -1,138 +0,0 @@ -#!/usr/bin/env {{ python_basename }} -# -# Description: Extract chronyd metrics from chronyc -c. -# Author: Aanchal Malhotra -# -# Works with chrony version 2.4 and higher -# -# this is from: https://www.mail-archive.com/chrony-users@chrony.tuxfamily.org/msg02179.html - -import subprocess -import sys - -chrony_sourcestats_cmd = ['chronyc', '-n', '-c', 'sourcestats'] -chrony_source_cmd = ['chronyc', '-n', '-c', 'sources'] -chrony_tracking_cmd = ['chronyc', '-n', '-c', 'tracking'] - -metrics_fields = [ - "Name/IP Address", - "NP", - "NR", - "Span", - "Frequency", - "Freq Skew", - "Offset", - "Std Dev"] - -status_types = {'x': 0, '?': 1, '-': 2, '+': 3, '*': 4} - -metrics_source = { - "*": "synchronized (system peer)", - "+": "synchronized", - "?": "unreachable", - "x": "Falseticker", - "-": "reference clock"} - -metrics_mode = { - '^': "server", - '=': "peer", - "#": "reference clock"} - - -def get_cmdoutput(command): - proc = subprocess.Popen(command, stdout=subprocess.PIPE) - out, err = proc.communicate() - return_code = proc.poll() - if return_code: - raise RuntimeError('Call to "{}" returned error: \ - {}'.format(command, return_code)) - return out.decode("utf-8") - - -def printPrometheusformat(metric, values): - print("# HELP chronyd_%s chronyd metric for %s" % (metric, metric)) - print("# TYPE chronyd_%s gauge" % (metric)) - for labels in values: - if labels is None: - print("chronyd_%s %f" % (metric, values[labels])) - else: - print("chronyd_%s{{ '{%' }}s} %f" % (metric, labels, values[labels])) - - -def printPrometheusscalar(metric, value): - print("# HELP chronyd_%s chronyd metric for %s" % (metric, metric)) - print("# TYPE chronyd_%s gauge" % (metric)) - print("chronyd_%s %f" % (metric, value)) - - -def printPrometheusEnum(metric, name): - print("# HELP chronyd_%s enum for %s" % (metric, metric)) - print("# TYPE chronyd_%s gauge" % (metric)) - print("chronyd_%s{value=\"%s\"} 1" % (metric, name)) - - -def weight(value): - val_int = int(value, 8) - return bin(val_int).count('1')/8.0 - - -def main(argv): - peer_status_metrics = {} - peer_reach_metrics = {} - offset_metrics = {} - freq_skew_metrics = {} - freq_metrics = {} - std_dev_metrics = {} - chrony_sourcestats = get_cmdoutput(chrony_sourcestats_cmd) - for line in chrony_sourcestats.split('\n'): - if (len(line)) > 0: - x = line.split(',') - common_labels = "remote=\"%s\"" % (x[0]) - freq_metrics[common_labels] = float(x[4]) - freq_skew_metrics[common_labels] = float(x[5]) - std_dev_metrics[common_labels] = float(x[7]) - - printPrometheusformat('freq_skew_ppm', freq_skew_metrics) - printPrometheusformat('freq_ppm', freq_metrics) - printPrometheusformat('std_dev_seconds', std_dev_metrics) - - chrony_source = get_cmdoutput(chrony_source_cmd) - for line in chrony_source.split('\n'): - if (len(line)) > 0: - x = line.split(',') - stratum = x[3] - reach = x[5] - mode = metrics_mode[x[0]] - common_labels = "remote=\"%s\"" % (x[2]) - peer_labels = "%s,stratum=\"%s\",mode=\"%s\"" % ( - common_labels, - stratum, - mode, - ) - peer_status_metrics[peer_labels] = float(status_types[x[1]]) - peer_reach_metrics[peer_labels] = weight(reach) - offset_metrics[common_labels] = float(x[8]) - - printPrometheusformat('peer_status', peer_status_metrics) - printPrometheusformat('offset_seconds', offset_metrics) - printPrometheusformat('peer_reachable', peer_reach_metrics) - - chrony_tracking_stats = get_cmdoutput(chrony_tracking_cmd).rstrip() - fields = chrony_tracking_stats.split(",") - printPrometheusEnum("tracking_source", fields[1]) - printPrometheusscalar("tracking_stratum", float(fields[2])) - printPrometheusscalar("tracking_ref_time", float(fields[3])) - printPrometheusscalar("tracking_system_time", float(fields[4])) - printPrometheusscalar("tracking_last_offset", float(fields[5])) - printPrometheusscalar("tracking_rms_offset", float(fields[6])) - printPrometheusscalar("tracking_frequency_error", float(fields[7])) - printPrometheusscalar("tracking_frequency_residual", float(fields[8])) - printPrometheusscalar("tracking_frequency_skew", float(fields[9])) - printPrometheusscalar("tracking_root_delay", float(fields[10])) - printPrometheusscalar("tracking_root_dispersion", float(fields[11])) - printPrometheusscalar("tracking_update_interval", float(fields[12])) - printPrometheusEnum("tracking_leap_status", fields[13]) - - -if __name__ == "__main__": - main(sys.argv[1:]) diff --git a/roles/monitoring/prometheus/exporter/node/templates/textfile-collector-scripts/chrony.service.j2 b/roles/monitoring/prometheus/exporter/node/templates/textfile-collector-scripts/chrony.service.j2 deleted file mode 100644 index 49b15185..00000000 --- a/roles/monitoring/prometheus/exporter/node/templates/textfile-collector-scripts/chrony.service.j2 +++ /dev/null @@ -1,33 +0,0 @@ -[Unit] -Description=Promethues node exporter textfile collector chrony - -[Service] -Type=oneshot -Environment=TMPDIR=/var/lib/prometheus-node-exporter/textfile-collector -ExecStart=bash -o pipefail -c "/usr/local/share/prometheus-node-exporter/chrony | sponge /var/lib/prometheus-node-exporter/textfile-collector/chrony.prom" -TimeoutStartSec=30s - -# systemd hardening-options -AmbientCapabilities=CAP_DAC_OVERRIDE -CapabilityBoundingSet=CAP_DAC_OVERRIDE -DeviceAllow=/dev/null rw -DevicePolicy=strict -LockPersonality=true -MemoryDenyWriteExecute=true -NoNewPrivileges=true -PrivateDevices=true -PrivateTmp=true -ProtectControlGroups=true -ProtectHome=true -ProtectKernelModules=true -ProtectKernelTunables=true -ProtectSystem=strict -ReadWritePaths=/var/lib/prometheus-node-exporter/textfile-collector /var/run/chrony -RemoveIPC=true -RestrictNamespaces=true -RestrictRealtime=true -RestrictAddressFamilies=AF_UNIX -SystemCallArchitectures=native - -[Install] -WantedBy=multi-user.target diff --git a/roles/monitoring/prometheus/exporter/node/templates/textfile-collector-scripts/chrony.timer.j2 b/roles/monitoring/prometheus/exporter/node/templates/textfile-collector-scripts/chrony.timer.j2 deleted file mode 100644 index eecc70e2..00000000 --- a/roles/monitoring/prometheus/exporter/node/templates/textfile-collector-scripts/chrony.timer.j2 +++ /dev/null @@ -1,9 +0,0 @@ -[Unit] -Description=Promethues node exporter textfile collector chrony - -[Timer] -OnBootSec=40s -OnUnitActiveSec=2min - -[Install] -WantedBy=timers.target diff --git a/roles/monitoring/prometheus/server/defaults/main/main.yml b/roles/monitoring/prometheus/server/defaults/main/main.yml index d778bad8..1e0dcf32 100644 --- a/roles/monitoring/prometheus/server/defaults/main/main.yml +++ b/roles/monitoring/prometheus/server/defaults/main/main.yml @@ -22,6 +22,7 @@ prometheus_server_rules: prometheus: "{{ prometheus_server_rules_prometheus + ((prometheus_server_alertmanager is defined) | ternary(prometheus_server_rules_prometheus_alertmanager, [])) + prometheus_server_rules_prometheus_extra }}" node: "{{ prometheus_server_rules_node + prometheus_server_rules_node_extra }}" openwrt: "{{ prometheus_server_rules_openwrt + prometheus_server_rules_openwrt_extra }}" + chrony: "{{ prometheus_server_rules_chrony + prometheus_server_rules_chrony_extra }}" nut: "{{ prometheus_server_rules_nut + prometheus_server_rules_nut_extra }}" nut/ups: "{{ prometheus_server_rules_nut__ups + prometheus_server_rules_nut__ups_extra }}" blackbox: "{{ prometheus_server_rules_blackbox + prometheus_server_rules_blackbox_extra }}" diff --git a/roles/monitoring/prometheus/server/defaults/main/rules_chrony.yml b/roles/monitoring/prometheus/server/defaults/main/rules_chrony.yml new file mode 100644 index 00000000..e845a60b --- /dev/null +++ b/roles/monitoring/prometheus/server/defaults/main/rules_chrony.yml @@ -0,0 +1,14 @@ +--- +prometheus_server_rules_chrony_extra: [] +prometheus_server_rules_chrony: + - record: instance:chrony_clock_error_seconds:abs + expr: abs(chrony_tracking_last_offset_seconds) + chrony_tracking_root_dispersion_seconds + (0.5 * chrony_tracking_root_delay_seconds) + + - alert: ChronyUnreachable + expr: chrony_up == 0 + for: 0m + labels: + severity: critical + annotations: + summary: Unable to scrape chrony metrics (instance {{ '{{' }} $labels.instance {{ '}}' }}) + description: "The chrony process might have crashed.\n VALUE = {{ '{{' }} $value {{ '}}' }}\n LABELS = {{ '{{' }} $labels {{ '}}' }}" -- cgit v1.2.3