Add provisioned Grafana dashboard for Yealink IP phones

Phones are ~99% of this Loki's log volume and nothing surfaced them. Adds a file-provisioned dashboard (uid yealink-phones, folder "Network") plus the provider config and the two grafana bind mounts it needs. Built against a 24h baseline of real handset traffic; all 14 panel queries were executed against live Loki before commit. Three findings shaped it, each of which contradicts the obvious reading of the data: - severity=emergency is NOT an emergency. All 24 emergency lines are the phone printing its own log-level table at boot ("sys log :type=1,E=3,W=4,N=5,I=6,D=7") — Yealink emits its logging config at level 0. Alerting on it would be 100% false positive. Filtered out. - "LSYS<3+error> rtpcap get len not enough" is 467 of 830 error lines (56%) — a noise floor, not a fault. Excluded from every "actionable" panel. - The hostname label cannot identify a device: its values are Yealink subsystems (sua, GUI, sys, cfg, ipp, dev, WEB, ATP), because the phones put the subsystem in the RFC3164 HOSTNAME field. Everything keys on source_ip. Yealink lines are separated from switch/router traffic on the same Loki by the |~ "<[0-7][+]" module marker (26,450 Yealink vs 33 non-Yealink lines in the baseline). Thresholds are seeded from measured 24h counts: check passwd err 21 (auth), data_task schedule time out 117 (handset scheduler overrunning its 30s threshold — the best "phone is unwell" proxy), tftp/provisioning failures 3, Register: update server 175 (SIP beat, 120s period), DHCP lease 9. Caveat: only two handsets report today, so thresholds are baselined on a 2-phone sample and want revisiting once the fleet is onboarded. Dashboards are file-provisioned, so UI edits are not persisted — change the JSON here and redeploy. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-07-15 15:53:11 +03:00
parent 9809cddbdf
commit 13b1887a16
3 changed files with 559 additions and 0 deletions
@@ -53,6 +53,11 @@ services:
    volumes:
      - /mnt/containers/logging/grafana-data:/var/lib/grafana:Z
      - /mnt/containers/logging/grafana-config/provisioning/datasources:/etc/grafana/provisioning/datasources:Z
+      # Dashboard provisioning: the provider config below points at the dashboards
+      # dir; both are file-provisioned, so UI edits are not persisted — edit the
+      # JSON in git and redeploy.
+      - /mnt/containers/logging/grafana-config/provisioning/dashboards:/etc/grafana/provisioning/dashboards:Z
+      - /mnt/containers/logging/grafana-config/dashboards:/var/lib/grafana/dashboards:Z
    networks:
      - logging-nw
      - reverseproxy-nw
@@ -0,0 +1,537 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": { "type": "grafana", "uid": "-- Grafana --" },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "type": "text",
+      "title": "How to read this dashboard",
+      "gridPos": { "h": 4, "w": 24, "x": 0, "y": 0 },
+      "id": 100,
+      "options": {
+        "mode": "markdown",
+        "content": "**Device identity is `source_ip`, not `hostname`.** Yealink phones put their internal *subsystem* (`sua`, `GUI`, `cfg`, `sys`, `ipp`…) in the syslog HOSTNAME field, so the `hostname` label cannot identify a device. Every panel here keys on `source_ip`.\n\n**Two things that look like alarms and are not.** `severity=emergency` is the phone printing its own log-level table at boot (`sys log :type=1,E=3,W=4,N=5,I=6,D=7`) — it is emitted at level 0 and means nothing; it is excluded here. `LSYS<3+error> rtpcap get len not enough` was **56% of all error lines** in the 24 h baseline — it is a noise floor and is excluded from every \"actionable\" panel below.\n\n**Panels filter to Yealink-format lines** via the `<N+level>` module marker, so switches and routers sharing this Loki do not pollute the phone view."
+      }
+    },
+
+    {
+      "type": "stat",
+      "title": "Phones reporting",
+      "description": "Distinct source_ip emitting Yealink-format syslog in the selected range.",
+      "gridPos": { "h": 4, "w": 4, "x": 0, "y": 4 },
+      "id": 1,
+      "datasource": { "type": "loki", "uid": "${datasource}" },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "loki", "uid": "${datasource}" },
+          "expr": "count(count by (source_ip) (count_over_time({job=\"syslog\", source_ip=~\"$phone\"} |~ \"<[0-7][+]\" [$__range])))",
+          "queryType": "instant"
+        }
+      ],
+      "options": {
+        "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
+        "colorMode": "value",
+        "graphMode": "none",
+        "textMode": "auto",
+        "justifyMode": "auto"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "color": { "mode": "fixed", "fixedColor": "text" },
+          "mappings": []
+        },
+        "overrides": []
+      }
+    },
+    {
+      "type": "stat",
+      "title": "Log lines",
+      "description": "Total Yealink-format syslog lines in range.",
+      "gridPos": { "h": 4, "w": 4, "x": 4, "y": 4 },
+      "id": 2,
+      "datasource": { "type": "loki", "uid": "${datasource}" },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "loki", "uid": "${datasource}" },
+          "expr": "sum(count_over_time({job=\"syslog\", source_ip=~\"$phone\"} |~ \"<[0-7][+]\" [$__range]))",
+          "queryType": "instant"
+        }
+      ],
+      "options": {
+        "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
+        "colorMode": "value",
+        "graphMode": "none",
+        "textMode": "auto"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "color": { "mode": "fixed", "fixedColor": "text" }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "type": "stat",
+      "title": "Actionable errors",
+      "description": "severity=error, excluding the 'rtpcap get len not enough' noise floor (56% of raw errors in baseline).",
+      "gridPos": { "h": 4, "w": 4, "x": 8, "y": 4 },
+      "id": 3,
+      "datasource": { "type": "loki", "uid": "${datasource}" },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "loki", "uid": "${datasource}" },
+          "expr": "sum(count_over_time({job=\"syslog\", source_ip=~\"$phone\", severity=\"error\"} !~ \"rtpcap get len not enough\" [$__range]))",
+          "queryType": "instant"
+        }
+      ],
+      "options": {
+        "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
+        "colorMode": "value",
+        "graphMode": "none",
+        "textMode": "auto"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "color": { "mode": "thresholds" },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "yellow", "value": 200 },
+              { "color": "red", "value": 800 }
+            ]
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "type": "stat",
+      "title": "Auth failures",
+      "description": "CUIT 'check passwd err' — phone web/UI credential rejections. Baseline: 21 / 24 h.",
+      "gridPos": { "h": 4, "w": 4, "x": 12, "y": 4 },
+      "id": 4,
+      "datasource": { "type": "loki", "uid": "${datasource}" },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "loki", "uid": "${datasource}" },
+          "expr": "sum(count_over_time({job=\"syslog\", source_ip=~\"$phone\"} |~ \"check passwd err\" [$__range]))",
+          "queryType": "instant"
+        }
+      ],
+      "options": {
+        "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
+        "colorMode": "value",
+        "graphMode": "none",
+        "textMode": "auto"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "color": { "mode": "thresholds" },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "yellow", "value": 25 },
+              { "color": "red", "value": 100 }
+            ]
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "type": "stat",
+      "title": "Provisioning failures",
+      "description": "ATP 'tftp to file failed' and SYS 'no config Provisioning Server'. Any non-zero value is worth a look — baseline was 3 / 24 h.",
+      "gridPos": { "h": 4, "w": 4, "x": 16, "y": 4 },
+      "id": 5,
+      "datasource": { "type": "loki", "uid": "${datasource}" },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "loki", "uid": "${datasource}" },
+          "expr": "sum(count_over_time({job=\"syslog\", source_ip=~\"$phone\"} |~ \"tftp to file failed|no config Provisioning Server\" [$__range]))",
+          "queryType": "instant"
+        }
+      ],
+      "options": {
+        "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
+        "colorMode": "value",
+        "graphMode": "none",
+        "textMode": "auto"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "color": { "mode": "thresholds" },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "yellow", "value": 1 },
+              { "color": "red", "value": 20 }
+            ]
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "type": "stat",
+      "title": "Scheduling timeouts",
+      "description": "DEV 'data_task schedule time out' — the phone's internal task scheduler overran its 30 s threshold. A rising count means the handset is struggling, and is the closest thing here to a 'phone is unwell' signal. Baseline: 117 / 24 h.",
+      "gridPos": { "h": 4, "w": 4, "x": 20, "y": 4 },
+      "id": 6,
+      "datasource": { "type": "loki", "uid": "${datasource}" },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "loki", "uid": "${datasource}" },
+          "expr": "sum(count_over_time({job=\"syslog\", source_ip=~\"$phone\"} |~ \"data_task schedule time out\" [$__range]))",
+          "queryType": "instant"
+        }
+      ],
+      "options": {
+        "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
+        "colorMode": "value",
+        "graphMode": "none",
+        "textMode": "auto"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "color": { "mode": "thresholds" },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "yellow", "value": 150 },
+              { "color": "red", "value": 500 }
+            ]
+          }
+        },
+        "overrides": []
+      }
+    },
+
+    {
+      "type": "timeseries",
+      "title": "Log rate by phone",
+      "description": "Lines/sec per device. A phone that goes flat has stopped sending syslog — that is either a dead handset, a reboot, or syslog switched off on the device (it silently defaults to disabled).",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
+      "id": 7,
+      "datasource": { "type": "loki", "uid": "${datasource}" },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "loki", "uid": "${datasource}" },
+          "expr": "sum by (source_ip) (rate({job=\"syslog\", source_ip=~\"$phone\"} |~ \"<[0-7][+]\" [$__interval]))",
+          "queryType": "range",
+          "legendFormat": "{{source_ip}}"
+        }
+      ],
+      "options": {
+        "legend": { "displayMode": "table", "placement": "right", "showLegend": true, "calcs": ["mean", "max"] },
+        "tooltip": { "mode": "multi", "sort": "desc" }
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "logs/s (calculated)",
+          "custom": {
+            "drawStyle": "line",
+            "lineWidth": 1,
+            "fillOpacity": 10,
+            "showPoints": "never",
+            "spanNulls": false
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "type": "timeseries",
+      "title": "Severity mix (emergency excluded — see notes)",
+      "description": "severity=emergency is deliberately filtered out: on Yealink it is the boot-time log-level table, not an incident.",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
+      "id": 8,
+      "datasource": { "type": "loki", "uid": "${datasource}" },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "loki", "uid": "${datasource}" },
+          "expr": "sum by (severity) (count_over_time({job=\"syslog\", source_ip=~\"$phone\", severity!=\"emergency\"} |~ \"<[0-7][+]\" [$__interval]))",
+          "queryType": "range",
+          "legendFormat": "{{severity}}"
+        }
+      ],
+      "options": {
+        "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true },
+        "tooltip": { "mode": "multi", "sort": "desc" }
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "custom": {
+            "drawStyle": "bars",
+            "lineWidth": 0,
+            "fillOpacity": 70,
+            "stacking": { "mode": "normal", "group": "A" },
+            "showPoints": "never"
+          }
+        },
+        "overrides": [
+          { "matcher": { "id": "byName", "options": "error" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } }] },
+          { "matcher": { "id": "byName", "options": "warning" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "orange" } }] },
+          { "matcher": { "id": "byName", "options": "notice" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "blue" } }] },
+          { "matcher": { "id": "byName", "options": "informational" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "green" } }] }
+        ]
+      }
+    },
+
+    {
+      "type": "timeseries",
+      "title": "Actionable errors by Yealink module",
+      "description": "Module tag parsed out of the message body (it is not a label). CFG = config attribute lookups, DEV = scheduler, CUIT = auth, ATP = provisioning/transfer, WEB = web UI.",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 },
+      "id": 9,
+      "datasource": { "type": "loki", "uid": "${datasource}" },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "loki", "uid": "${datasource}" },
+          "expr": "sum by (module) (count_over_time({job=\"syslog\", source_ip=~\"$phone\", severity=\"error\"} !~ \"rtpcap get len not enough\" | regexp \"(?P<module>[A-Z]{2,5}) *<3[+]error\" [$__interval]))",
+          "queryType": "range",
+          "legendFormat": "{{module}}"
+        }
+      ],
+      "options": {
+        "legend": { "displayMode": "table", "placement": "right", "showLegend": true, "calcs": ["sum"] },
+        "tooltip": { "mode": "multi", "sort": "desc" }
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "custom": {
+            "drawStyle": "bars",
+            "lineWidth": 0,
+            "fillOpacity": 80,
+            "stacking": { "mode": "normal", "group": "A" },
+            "showPoints": "never"
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "type": "bargauge",
+      "title": "Actionable errors by phone",
+      "description": "Which handset is actually generating the errors, noise floor removed.",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 },
+      "id": 10,
+      "datasource": { "type": "loki", "uid": "${datasource}" },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "loki", "uid": "${datasource}" },
+          "expr": "sum by (source_ip) (count_over_time({job=\"syslog\", source_ip=~\"$phone\", severity=\"error\"} !~ \"rtpcap get len not enough\" [$__range]))",
+          "queryType": "instant",
+          "legendFormat": "{{source_ip}}"
+        }
+      ],
+      "options": {
+        "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
+        "displayMode": "gradient",
+        "orientation": "horizontal",
+        "showUnfilled": true
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "color": { "mode": "continuous-GrYlRd" }
+        },
+        "overrides": []
+      }
+    },
+
+    {
+      "type": "logs",
+      "title": "Signals that matter — auth, provisioning, scheduler",
+      "description": "The three narrow patterns confirmed to carry real operational meaning on these handsets. This is the panel to read first when a phone is reported faulty.",
+      "gridPos": { "h": 10, "w": 24, "x": 0, "y": 24 },
+      "id": 11,
+      "datasource": { "type": "loki", "uid": "${datasource}" },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "loki", "uid": "${datasource}" },
+          "expr": "{job=\"syslog\", source_ip=~\"$phone\"} |~ \"check passwd err|tftp to file failed|no config Provisioning Server|data_task schedule time out\"",
+          "queryType": "range"
+        }
+      ],
+      "options": {
+        "showTime": true,
+        "showLabels": true,
+        "showCommonLabels": false,
+        "wrapLogMessage": true,
+        "prettifyLogMessage": false,
+        "enableLogDetails": true,
+        "dedupStrategy": "none",
+        "sortOrder": "Descending"
+      }
+    },
+
+    {
+      "type": "logs",
+      "title": "All actionable errors (rtpcap noise floor removed)",
+      "gridPos": { "h": 10, "w": 24, "x": 0, "y": 34 },
+      "id": 12,
+      "datasource": { "type": "loki", "uid": "${datasource}" },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "loki", "uid": "${datasource}" },
+          "expr": "{job=\"syslog\", source_ip=~\"$phone\", severity=\"error\"} !~ \"rtpcap get len not enough\"",
+          "queryType": "range"
+        }
+      ],
+      "options": {
+        "showTime": true,
+        "showLabels": true,
+        "showCommonLabels": false,
+        "wrapLogMessage": true,
+        "enableLogDetails": true,
+        "dedupStrategy": "exact",
+        "sortOrder": "Descending"
+      }
+    },
+
+    {
+      "type": "timeseries",
+      "title": "SIP registration refresh",
+      "description": "REG 'Register: update server' events. Registration period observed at 120 s, so a healthy phone should show a steady beat here; gaps mean the handset lost its registration to the PBX (192.168.0.8).",
+      "gridPos": { "h": 7, "w": 12, "x": 0, "y": 44 },
+      "id": 13,
+      "datasource": { "type": "loki", "uid": "${datasource}" },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "loki", "uid": "${datasource}" },
+          "expr": "sum by (source_ip) (count_over_time({job=\"syslog\", source_ip=~\"$phone\"} |~ \"Register: update server\" [$__interval]))",
+          "queryType": "range",
+          "legendFormat": "{{source_ip}}"
+        }
+      ],
+      "options": {
+        "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true },
+        "tooltip": { "mode": "multi" }
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "custom": {
+            "drawStyle": "points",
+            "pointSize": 5,
+            "lineWidth": 1,
+            "fillOpacity": 0,
+            "showPoints": "always"
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "type": "logs",
+      "title": "DHCP lease events",
+      "description": "Lease acquisition/renewal as the phone sees it. Useful when a handset is 'connected but dead' — cross-reference against the router's DHCP log.",
+      "gridPos": { "h": 7, "w": 12, "x": 12, "y": 44 },
+      "id": 14,
+      "datasource": { "type": "loki", "uid": "${datasource}" },
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": { "type": "loki", "uid": "${datasource}" },
+          "expr": "{job=\"syslog\", source_ip=~\"$phone\"} |~ \"lease of .* obtained|lease renew\"",
+          "queryType": "range"
+        }
+      ],
+      "options": {
+        "showTime": true,
+        "showLabels": false,
+        "wrapLogMessage": true,
+        "enableLogDetails": true,
+        "dedupStrategy": "none",
+        "sortOrder": "Descending"
+      }
+    }
+  ],
+  "refresh": "1m",
+  "schemaVersion": 39,
+  "tags": ["yealink", "phones", "syslog", "02-tekhnohim"],
+  "templating": {
+    "list": [
+      {
+        "name": "datasource",
+        "label": "Datasource",
+        "type": "datasource",
+        "query": "loki",
+        "current": {},
+        "hide": 0,
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false
+      },
+      {
+        "name": "phone",
+        "label": "Phone (source_ip)",
+        "type": "query",
+        "datasource": { "type": "loki", "uid": "${datasource}" },
+        "definition": "label_values({job=\"syslog\"}, source_ip)",
+        "query": {
+          "label": "source_ip",
+          "refId": "LokiVariableQueryEditor-VariableQuery",
+          "stream": "{job=\"syslog\"}",
+          "type": 1
+        },
+        "current": { "selected": true, "text": ["All"], "value": ["$__all"] },
+        "includeAll": true,
+        "multi": true,
+        "allValue": ".*",
+        "hide": 0,
+        "refresh": 2,
+        "sort": 1,
+        "skipUrlSync": false
+      }
+    ]
+  },
+  "time": { "from": "now-24h", "to": "now" },
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "Yealink IP phones — syslog health",
+  "uid": "yealink-phones",
+  "version": 1,
+  "weekStart": ""
+}
@@ -0,0 +1,17 @@
+apiVersion: 1
+
+# Loads every dashboard JSON from /var/lib/grafana/dashboards (bind-mounted from
+# this repo's logging/grafana-dashboards/). Dashboards are file-provisioned, so
+# they are recreated on every container start and edits made in the UI are not
+# persisted — change the JSON in git and redeploy instead.
+providers:
+  - name: 'logging-dashboards'
+    orgId: 1
+    folder: 'Network'
+    type: file
+    disableDeletion: false
+    updateIntervalSeconds: 30
+    allowUiUpdates: false
+    options:
+      path: /var/lib/grafana/dashboards
+      foldersFromFilesStructure: false