diff --git a/logging/docker-compose.yaml b/logging/docker-compose.yaml index c792eda..d0229e8 100644 --- a/logging/docker-compose.yaml +++ b/logging/docker-compose.yaml @@ -53,6 +53,11 @@ services: volumes: - /mnt/containers/logging/grafana-data:/var/lib/grafana:Z - /mnt/containers/logging/grafana-config/provisioning/datasources:/etc/grafana/provisioning/datasources:Z + # Dashboard provisioning: the provider config below points at the dashboards + # dir; both are file-provisioned, so UI edits are not persisted — edit the + # JSON in git and redeploy. + - /mnt/containers/logging/grafana-config/provisioning/dashboards:/etc/grafana/provisioning/dashboards:Z + - /mnt/containers/logging/grafana-config/dashboards:/var/lib/grafana/dashboards:Z networks: - logging-nw - reverseproxy-nw diff --git a/logging/grafana-dashboards/yealink-phones.json b/logging/grafana-dashboards/yealink-phones.json new file mode 100644 index 0000000..a73544d --- /dev/null +++ b/logging/grafana-dashboards/yealink-phones.json @@ -0,0 +1,537 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { "type": "grafana", "uid": "-- Grafana --" }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [], + "liveNow": false, + "panels": [ + { + "type": "text", + "title": "How to read this dashboard", + "gridPos": { "h": 4, "w": 24, "x": 0, "y": 0 }, + "id": 100, + "options": { + "mode": "markdown", + "content": "**Device identity is `source_ip`, not `hostname`.** Yealink phones put their internal *subsystem* (`sua`, `GUI`, `cfg`, `sys`, `ipp`…) in the syslog HOSTNAME field, so the `hostname` label cannot identify a device. Every panel here keys on `source_ip`.\n\n**Two things that look like alarms and are not.** `severity=emergency` is the phone printing its own log-level table at boot (`sys log :type=1,E=3,W=4,N=5,I=6,D=7`) — it is emitted at level 0 and means nothing; it is excluded here. `LSYS<3+error> rtpcap get len not enough` was **56% of all error lines** in the 24 h baseline — it is a noise floor and is excluded from every \"actionable\" panel below.\n\n**Panels filter to Yealink-format lines** via the `` module marker, so switches and routers sharing this Loki do not pollute the phone view." + } + }, + + { + "type": "stat", + "title": "Phones reporting", + "description": "Distinct source_ip emitting Yealink-format syslog in the selected range.", + "gridPos": { "h": 4, "w": 4, "x": 0, "y": 4 }, + "id": 1, + "datasource": { "type": "loki", "uid": "${datasource}" }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "loki", "uid": "${datasource}" }, + "expr": "count(count by (source_ip) (count_over_time({job=\"syslog\", source_ip=~\"$phone\"} |~ \"<[0-7][+]\" [$__range])))", + "queryType": "instant" + } + ], + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "colorMode": "value", + "graphMode": "none", + "textMode": "auto", + "justifyMode": "auto" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { "mode": "fixed", "fixedColor": "text" }, + "mappings": [] + }, + "overrides": [] + } + }, + { + "type": "stat", + "title": "Log lines", + "description": "Total Yealink-format syslog lines in range.", + "gridPos": { "h": 4, "w": 4, "x": 4, "y": 4 }, + "id": 2, + "datasource": { "type": "loki", "uid": "${datasource}" }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "loki", "uid": "${datasource}" }, + "expr": "sum(count_over_time({job=\"syslog\", source_ip=~\"$phone\"} |~ \"<[0-7][+]\" [$__range]))", + "queryType": "instant" + } + ], + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "colorMode": "value", + "graphMode": "none", + "textMode": "auto" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { "mode": "fixed", "fixedColor": "text" } + }, + "overrides": [] + } + }, + { + "type": "stat", + "title": "Actionable errors", + "description": "severity=error, excluding the 'rtpcap get len not enough' noise floor (56% of raw errors in baseline).", + "gridPos": { "h": 4, "w": 4, "x": 8, "y": 4 }, + "id": 3, + "datasource": { "type": "loki", "uid": "${datasource}" }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "loki", "uid": "${datasource}" }, + "expr": "sum(count_over_time({job=\"syslog\", source_ip=~\"$phone\", severity=\"error\"} !~ \"rtpcap get len not enough\" [$__range]))", + "queryType": "instant" + } + ], + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "colorMode": "value", + "graphMode": "none", + "textMode": "auto" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 200 }, + { "color": "red", "value": 800 } + ] + } + }, + "overrides": [] + } + }, + { + "type": "stat", + "title": "Auth failures", + "description": "CUIT 'check passwd err' — phone web/UI credential rejections. Baseline: 21 / 24 h.", + "gridPos": { "h": 4, "w": 4, "x": 12, "y": 4 }, + "id": 4, + "datasource": { "type": "loki", "uid": "${datasource}" }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "loki", "uid": "${datasource}" }, + "expr": "sum(count_over_time({job=\"syslog\", source_ip=~\"$phone\"} |~ \"check passwd err\" [$__range]))", + "queryType": "instant" + } + ], + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "colorMode": "value", + "graphMode": "none", + "textMode": "auto" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 25 }, + { "color": "red", "value": 100 } + ] + } + }, + "overrides": [] + } + }, + { + "type": "stat", + "title": "Provisioning failures", + "description": "ATP 'tftp to file failed' and SYS 'no config Provisioning Server'. Any non-zero value is worth a look — baseline was 3 / 24 h.", + "gridPos": { "h": 4, "w": 4, "x": 16, "y": 4 }, + "id": 5, + "datasource": { "type": "loki", "uid": "${datasource}" }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "loki", "uid": "${datasource}" }, + "expr": "sum(count_over_time({job=\"syslog\", source_ip=~\"$phone\"} |~ \"tftp to file failed|no config Provisioning Server\" [$__range]))", + "queryType": "instant" + } + ], + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "colorMode": "value", + "graphMode": "none", + "textMode": "auto" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 1 }, + { "color": "red", "value": 20 } + ] + } + }, + "overrides": [] + } + }, + { + "type": "stat", + "title": "Scheduling timeouts", + "description": "DEV 'data_task schedule time out' — the phone's internal task scheduler overran its 30 s threshold. A rising count means the handset is struggling, and is the closest thing here to a 'phone is unwell' signal. Baseline: 117 / 24 h.", + "gridPos": { "h": 4, "w": 4, "x": 20, "y": 4 }, + "id": 6, + "datasource": { "type": "loki", "uid": "${datasource}" }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "loki", "uid": "${datasource}" }, + "expr": "sum(count_over_time({job=\"syslog\", source_ip=~\"$phone\"} |~ \"data_task schedule time out\" [$__range]))", + "queryType": "instant" + } + ], + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "colorMode": "value", + "graphMode": "none", + "textMode": "auto" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 150 }, + { "color": "red", "value": 500 } + ] + } + }, + "overrides": [] + } + }, + + { + "type": "timeseries", + "title": "Log rate by phone", + "description": "Lines/sec per device. A phone that goes flat has stopped sending syslog — that is either a dead handset, a reboot, or syslog switched off on the device (it silently defaults to disabled).", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, + "id": 7, + "datasource": { "type": "loki", "uid": "${datasource}" }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "loki", "uid": "${datasource}" }, + "expr": "sum by (source_ip) (rate({job=\"syslog\", source_ip=~\"$phone\"} |~ \"<[0-7][+]\" [$__interval]))", + "queryType": "range", + "legendFormat": "{{source_ip}}" + } + ], + "options": { + "legend": { "displayMode": "table", "placement": "right", "showLegend": true, "calcs": ["mean", "max"] }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "fieldConfig": { + "defaults": { + "unit": "logs/s (calculated)", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false + } + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Severity mix (emergency excluded — see notes)", + "description": "severity=emergency is deliberately filtered out: on Yealink it is the boot-time log-level table, not an incident.", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 }, + "id": 8, + "datasource": { "type": "loki", "uid": "${datasource}" }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "loki", "uid": "${datasource}" }, + "expr": "sum by (severity) (count_over_time({job=\"syslog\", source_ip=~\"$phone\", severity!=\"emergency\"} |~ \"<[0-7][+]\" [$__interval]))", + "queryType": "range", + "legendFormat": "{{severity}}" + } + ], + "options": { + "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "bars", + "lineWidth": 0, + "fillOpacity": 70, + "stacking": { "mode": "normal", "group": "A" }, + "showPoints": "never" + } + }, + "overrides": [ + { "matcher": { "id": "byName", "options": "error" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } }] }, + { "matcher": { "id": "byName", "options": "warning" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "orange" } }] }, + { "matcher": { "id": "byName", "options": "notice" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "blue" } }] }, + { "matcher": { "id": "byName", "options": "informational" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "green" } }] } + ] + } + }, + + { + "type": "timeseries", + "title": "Actionable errors by Yealink module", + "description": "Module tag parsed out of the message body (it is not a label). CFG = config attribute lookups, DEV = scheduler, CUIT = auth, ATP = provisioning/transfer, WEB = web UI.", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 }, + "id": 9, + "datasource": { "type": "loki", "uid": "${datasource}" }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "loki", "uid": "${datasource}" }, + "expr": "sum by (module) (count_over_time({job=\"syslog\", source_ip=~\"$phone\", severity=\"error\"} !~ \"rtpcap get len not enough\" | regexp \"(?P[A-Z]{2,5}) *<3[+]error\" [$__interval]))", + "queryType": "range", + "legendFormat": "{{module}}" + } + ], + "options": { + "legend": { "displayMode": "table", "placement": "right", "showLegend": true, "calcs": ["sum"] }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "bars", + "lineWidth": 0, + "fillOpacity": 80, + "stacking": { "mode": "normal", "group": "A" }, + "showPoints": "never" + } + }, + "overrides": [] + } + }, + { + "type": "bargauge", + "title": "Actionable errors by phone", + "description": "Which handset is actually generating the errors, noise floor removed.", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 }, + "id": 10, + "datasource": { "type": "loki", "uid": "${datasource}" }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "loki", "uid": "${datasource}" }, + "expr": "sum by (source_ip) (count_over_time({job=\"syslog\", source_ip=~\"$phone\", severity=\"error\"} !~ \"rtpcap get len not enough\" [$__range]))", + "queryType": "instant", + "legendFormat": "{{source_ip}}" + } + ], + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "displayMode": "gradient", + "orientation": "horizontal", + "showUnfilled": true + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { "mode": "continuous-GrYlRd" } + }, + "overrides": [] + } + }, + + { + "type": "logs", + "title": "Signals that matter — auth, provisioning, scheduler", + "description": "The three narrow patterns confirmed to carry real operational meaning on these handsets. This is the panel to read first when a phone is reported faulty.", + "gridPos": { "h": 10, "w": 24, "x": 0, "y": 24 }, + "id": 11, + "datasource": { "type": "loki", "uid": "${datasource}" }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "loki", "uid": "${datasource}" }, + "expr": "{job=\"syslog\", source_ip=~\"$phone\"} |~ \"check passwd err|tftp to file failed|no config Provisioning Server|data_task schedule time out\"", + "queryType": "range" + } + ], + "options": { + "showTime": true, + "showLabels": true, + "showCommonLabels": false, + "wrapLogMessage": true, + "prettifyLogMessage": false, + "enableLogDetails": true, + "dedupStrategy": "none", + "sortOrder": "Descending" + } + }, + + { + "type": "logs", + "title": "All actionable errors (rtpcap noise floor removed)", + "gridPos": { "h": 10, "w": 24, "x": 0, "y": 34 }, + "id": 12, + "datasource": { "type": "loki", "uid": "${datasource}" }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "loki", "uid": "${datasource}" }, + "expr": "{job=\"syslog\", source_ip=~\"$phone\", severity=\"error\"} !~ \"rtpcap get len not enough\"", + "queryType": "range" + } + ], + "options": { + "showTime": true, + "showLabels": true, + "showCommonLabels": false, + "wrapLogMessage": true, + "enableLogDetails": true, + "dedupStrategy": "exact", + "sortOrder": "Descending" + } + }, + + { + "type": "timeseries", + "title": "SIP registration refresh", + "description": "REG 'Register: update server' events. Registration period observed at 120 s, so a healthy phone should show a steady beat here; gaps mean the handset lost its registration to the PBX (192.168.0.8).", + "gridPos": { "h": 7, "w": 12, "x": 0, "y": 44 }, + "id": 13, + "datasource": { "type": "loki", "uid": "${datasource}" }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "loki", "uid": "${datasource}" }, + "expr": "sum by (source_ip) (count_over_time({job=\"syslog\", source_ip=~\"$phone\"} |~ \"Register: update server\" [$__interval]))", + "queryType": "range", + "legendFormat": "{{source_ip}}" + } + ], + "options": { + "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi" } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "points", + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "showPoints": "always" + } + }, + "overrides": [] + } + }, + { + "type": "logs", + "title": "DHCP lease events", + "description": "Lease acquisition/renewal as the phone sees it. Useful when a handset is 'connected but dead' — cross-reference against the router's DHCP log.", + "gridPos": { "h": 7, "w": 12, "x": 12, "y": 44 }, + "id": 14, + "datasource": { "type": "loki", "uid": "${datasource}" }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "loki", "uid": "${datasource}" }, + "expr": "{job=\"syslog\", source_ip=~\"$phone\"} |~ \"lease of .* obtained|lease renew\"", + "queryType": "range" + } + ], + "options": { + "showTime": true, + "showLabels": false, + "wrapLogMessage": true, + "enableLogDetails": true, + "dedupStrategy": "none", + "sortOrder": "Descending" + } + } + ], + "refresh": "1m", + "schemaVersion": 39, + "tags": ["yealink", "phones", "syslog", "02-tekhnohim"], + "templating": { + "list": [ + { + "name": "datasource", + "label": "Datasource", + "type": "datasource", + "query": "loki", + "current": {}, + "hide": 0, + "refresh": 1, + "regex": "", + "skipUrlSync": false + }, + { + "name": "phone", + "label": "Phone (source_ip)", + "type": "query", + "datasource": { "type": "loki", "uid": "${datasource}" }, + "definition": "label_values({job=\"syslog\"}, source_ip)", + "query": { + "label": "source_ip", + "refId": "LokiVariableQueryEditor-VariableQuery", + "stream": "{job=\"syslog\"}", + "type": 1 + }, + "current": { "selected": true, "text": ["All"], "value": ["$__all"] }, + "includeAll": true, + "multi": true, + "allValue": ".*", + "hide": 0, + "refresh": 2, + "sort": 1, + "skipUrlSync": false + } + ] + }, + "time": { "from": "now-24h", "to": "now" }, + "timepicker": {}, + "timezone": "browser", + "title": "Yealink IP phones — syslog health", + "uid": "yealink-phones", + "version": 1, + "weekStart": "" +} diff --git a/logging/grafana-provisioning/dashboards/dashboards.yaml b/logging/grafana-provisioning/dashboards/dashboards.yaml new file mode 100644 index 0000000..4ced255 --- /dev/null +++ b/logging/grafana-provisioning/dashboards/dashboards.yaml @@ -0,0 +1,17 @@ +apiVersion: 1 + +# Loads every dashboard JSON from /var/lib/grafana/dashboards (bind-mounted from +# this repo's logging/grafana-dashboards/). Dashboards are file-provisioned, so +# they are recreated on every container start and edits made in the UI are not +# persisted — change the JSON in git and redeploy instead. +providers: + - name: 'logging-dashboards' + orgId: 1 + folder: 'Network' + type: file + disableDeletion: false + updateIntervalSeconds: 30 + allowUiUpdates: false + options: + path: /var/lib/grafana/dashboards + foldersFromFilesStructure: false