From 1a42f89c1129744eae8ed52eb720506919c554cb Mon Sep 17 00:00:00 2001 From: pipistrello Date: Fri, 17 Jul 2026 14:19:32 +0300 Subject: [PATCH] grafana: separate phone health signals from firmware noise The Yealink dashboard treated nearly every level-3 line as actionable, making routine boot and firmware chatter look like fleet-wide failures.\n\nReplace generic error counts with recent LDAP, provisioning, and internal-database signals. Add dedicated operational trend and log panels, and relabel raw severity as diagnostic-only while retaining web-password, registration, and DHCP context.\n\nValidated all 17 LogQL targets directly against Loki 3.7.2. In the inspected 24-hour sample, provisioning had settled, LDAP failures remained active, and one handset reported a malformed internal database. --- .../grafana-dashboards/yealink-phones.json | 82 ++++++++++++------- 1 file changed, 52 insertions(+), 30 deletions(-) diff --git a/logging/grafana-dashboards/yealink-phones.json b/logging/grafana-dashboards/yealink-phones.json index 24da401..ad4c7a8 100644 --- a/logging/grafana-dashboards/yealink-phones.json +++ b/logging/grafana-dashboards/yealink-phones.json @@ -25,7 +25,7 @@ "id": 100, "options": { "mode": "markdown", - "content": "**Identity is the phone's MAC**, lifted out of the message body into a `mac` label by Alloy (Yealink `static.syslog.prepend_mac_address.enable`). Do **not** use `source_ip` as identity — phones are on DHCP with a 1200 s lease, so a re-IP looks like a new device and a recycled IP would inherit another phone's history. Only devices sending a MAC appear here, so switches sharing this Loki are excluded automatically. *Data from before the MAC rollout has no `mac` label and will not show.*\n\n**A silent phone is a healthy phone, not a missing one.** Most of the fleet runs at Yealink's default log level **3 (Error)** and only speaks when something is wrong. A couple of handsets are at level **6 (Informational)** and produce ~80% of all volume. These panels count *phones heard from*, never *phones alive* — and the registration/DHCP panels at the bottom can only ever populate for level-6 phones.\n\n**Two known noise sources are filtered out:** `severity=emergency` is Yealink printing its own log-level table at boot (`E=3,W=4,N=5,I=6,D=7`), not an incident; and `LSYS<3+error> rtpcap get len not enough` was 56% of all error lines in baseline. Neither means anything.\n\n**No colour thresholds on the counts — deliberately.** The fleet onboarded 2026-07-15, so there is no representative 24 h baseline to derive them from, and fleet-wide sums scale with phone count rather than with health. The panels are *comparative* (worst offenders) instead. Revisit once a full week of fleet data exists." + "content": "**Identity is the phone's MAC**, lifted out of the message body into a `mac` label by Alloy (Yealink `static.syslog.prepend_mac_address.enable`). Do **not** use `source_ip` as identity — phones are on DHCP with a 1200 s lease, so a re-IP looks like a new device and a recycled IP would inherit another phone's history. Only devices sending a MAC appear here, so switches sharing this Loki are excluded automatically. *Data from before the MAC rollout has no `mac` label and will not show.*\n\n**Yealink's `error` severity is diagnostic verbosity, not a fleet health verdict.** Firmware-internal messages such as missing optional DHCP attributes, GUI event targets, RTP capture reads, scheduler overruns near the built-in 30 s threshold, kernel register dumps, and provisioning-engine state account for most red lines. Use the narrow **Operational signals** panels first; use raw severity only when investigating one handset.\n\n**The top status cards are intentionally recent.** LDAP and internal-database cards cover the last 15 minutes; provisioning covers the last hour. This prevents a completed fleet rollout or reboot from looking like a current outage for the rest of a 24 h dashboard range. LDAP failures affect directory/search and caller-name lookup, not SIP calling itself.\n\n**A silent phone is not proven healthy or offline.** Most of the fleet runs at log level 3 and only sends selected errors. Registration and DHCP evidence at the bottom is available only from verbose level-6 phones. `severity=emergency` is also excluded from the severity chart because Yealink uses it while printing its boot-time log-level table, not to report an emergency." } }, @@ -77,8 +77,8 @@ }, { "type": "stat", - "title": "Phones with errors", - "description": "How many distinct handsets logged at least one actionable error (rtpcap noise floor excluded). A count of affected devices — unlike a fleet-wide error total, this does not inflate as phones are added.", + "title": "Active LDAP failures (15m)", + "description": "Distinct phones that could not bind/contact the LDAP directory in the last 15 minutes. This can break directory search and caller-name lookup, but does not by itself indicate a SIP calling outage.", "gridPos": { "h": 4, "w": 4, "x": 8, "y": 5 }, "id": 3, "datasource": { "type": "loki", "uid": "${datasource}" }, @@ -86,22 +86,22 @@ { "refId": "A", "datasource": { "type": "loki", "uid": "${datasource}" }, - "expr": "count(count by (mac) (count_over_time({job=\"syslog\", mac=~\"$phone\", severity=\"error\"} !~ \"rtpcap get len not enough\" [$__range])))", + "expr": "count(count by (mac) (count_over_time({job=\"syslog\", mac=~\"$phone\"} |~ \"ldap_sasl_bind_s|Can't contact LDAP server\" [15m])))", "queryType": "instant" } ], "options": { "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, - "colorMode": "none", + "colorMode": "value", "graphMode": "none", "textMode": "auto" }, - "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "fixed", "fixedColor": "text" } }, "overrides": [] } + "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 1 }] } }, "overrides": [] } }, { "type": "stat", - "title": "Phones failing provisioning", - "description": "Handsets hitting 'tftp to file failed' or 'no config Provisioning Server'. Any non-zero value is worth a look regardless of fleet size — this is a qualitative fault, not a magnitude, which is why it carries a threshold when the count panels deliberately do not.", + "title": "Recent provisioning failures (1h)", + "description": "Distinct phones with TFTP/provisioning download failures in the last hour. A short burst during reboot or fleet rollout is expected; a value that remains non-zero deserves investigation.", "gridPos": { "h": 4, "w": 4, "x": 12, "y": 5 }, "id": 4, "datasource": { "type": "loki", "uid": "${datasource}" }, @@ -109,7 +109,7 @@ { "refId": "A", "datasource": { "type": "loki", "uid": "${datasource}" }, - "expr": "count(count by (mac) (count_over_time({job=\"syslog\", mac=~\"$phone\"} |~ \"tftp to file failed|no config Provisioning Server\" [$__range])))", + "expr": "count(count by (mac) (count_over_time({job=\"syslog\", mac=~\"$phone\"} |~ \"tftp to file failed|no config Provisioning Server|DURL<3[+]error\" [1h])))", "queryType": "instant" } ], @@ -130,8 +130,8 @@ }, { "type": "stat", - "title": "Phones with auth failures", - "description": "Handsets logging CUIT 'check passwd err' — a credential rejection on the phone's own web/UI. Could be a mis-provisioned password or someone probing; read the log panel below rather than treating the number as a score.", + "title": "Web password rejects", + "description": "Phones logging CUIT 'check passwd err' in the selected range. This is a rejected login to the handset web/UI, not SIP authentication; occasional entries are usually operator mistakes or background probing.", "gridPos": { "h": 4, "w": 4, "x": 16, "y": 5 }, "id": 5, "datasource": { "type": "loki", "uid": "${datasource}" }, @@ -153,8 +153,8 @@ }, { "type": "stat", - "title": "Phones with scheduler timeouts", - "description": "Handsets logging DEV 'data_task schedule time out' — the phone's internal scheduler overran its 30 s threshold. The best available proxy for 'this handset is struggling'.", + "title": "Internal DB faults (15m)", + "description": "Phones reporting a malformed internal database or related GUI virtual-machine failures in the last 15 minutes. Unlike ordinary firmware chatter, a persistent value can affect local phone applications and should be checked.", "gridPos": { "h": 4, "w": 4, "x": 20, "y": 5 }, "id": 6, "datasource": { "type": "loki", "uid": "${datasource}" }, @@ -162,23 +162,23 @@ { "refId": "A", "datasource": { "type": "loki", "uid": "${datasource}" }, - "expr": "count(count by (mac) (count_over_time({job=\"syslog\", mac=~\"$phone\"} |~ \"data_task schedule time out\" [$__range])))", + "expr": "count(count by (mac) (count_over_time({job=\"syslog\", mac=~\"$phone\"} |~ \"database disk image is malformed|Null Virtual Machine pointer|Invalid scalar query\" [15m])))", "queryType": "instant" } ], "options": { "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, - "colorMode": "none", + "colorMode": "value", "graphMode": "none", "textMode": "auto" }, - "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "fixed", "fixedColor": "text" } }, "overrides": [] } + "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 1 }] } }, "overrides": [] } }, { "type": "bargauge", - "title": "Worst phones by actionable errors", - "description": "Comparative, not absolute — find the outlier rather than judge against a made-up number. Keyed on MAC, so a phone that changes IP stays the same bar.", + "title": "Phones by operational-signal events", + "description": "Top phones for the narrow signals this dashboard can interpret: LDAP connectivity, internal database faults, provisioning/download failures, and SIP DNS timeouts. Keyed on MAC so DHCP changes do not split identity.", "gridPos": { "h": 9, "w": 12, "x": 0, "y": 9 }, "id": 7, "datasource": { "type": "loki", "uid": "${datasource}" }, @@ -186,7 +186,7 @@ { "refId": "A", "datasource": { "type": "loki", "uid": "${datasource}" }, - "expr": "topk(10, sum by (mac) (count_over_time({job=\"syslog\", mac=~\"$phone\", severity=\"error\"} !~ \"rtpcap get len not enough\" [$__range])))", + "expr": "topk(10, sum by (mac) (count_over_time({job=\"syslog\", mac=~\"$phone\"} |~ \"ldap_sasl_bind_s|Can't contact LDAP server|database disk image is malformed|Null Virtual Machine pointer|Invalid scalar query|tftp to file failed|no config Provisioning Server|DURL<3[+]error|Timeout while contacting DNS servers\" [$__range])))", "queryType": "instant", "legendFormat": "{{mac}}" } @@ -226,8 +226,8 @@ { "type": "timeseries", - "title": "Actionable errors by Yealink module", - "description": "Module tag parsed from the message body (it is not a label). CFG = config attribute lookups, DEV = scheduler, CUIT = auth, ATP = provisioning/transfer, WEB = web UI, LSYS = RTP capture.", + "title": "Operational signals over time", + "description": "Narrow, interpretable signals separated from firmware error-severity noise. LDAP affects directory lookup; internal DB affects local apps; provisioning should settle after boot; SIP DNS timeouts can prevent registration.", "gridPos": { "h": 8, "w": 12, "x": 0, "y": 18 }, "id": 9, "datasource": { "type": "loki", "uid": "${datasource}" }, @@ -235,9 +235,30 @@ { "refId": "A", "datasource": { "type": "loki", "uid": "${datasource}" }, - "expr": "sum by (module) (count_over_time({job=\"syslog\", mac=~\"$phone\", severity=\"error\"} !~ \"rtpcap get len not enough\" | regexp \"(?P[A-Z]{2,5}) *<3[+]error\" [$__interval]))", + "expr": "sum(count_over_time({job=\"syslog\", mac=~\"$phone\"} |~ \"ldap_sasl_bind_s|Can't contact LDAP server\" [$__interval]))", "queryType": "range", - "legendFormat": "{{module}}" + "legendFormat": "LDAP directory" + }, + { + "refId": "B", + "datasource": { "type": "loki", "uid": "${datasource}" }, + "expr": "sum(count_over_time({job=\"syslog\", mac=~\"$phone\"} |~ \"database disk image is malformed|Null Virtual Machine pointer|Invalid scalar query\" [$__interval]))", + "queryType": "range", + "legendFormat": "Internal DB / GUI" + }, + { + "refId": "C", + "datasource": { "type": "loki", "uid": "${datasource}" }, + "expr": "sum(count_over_time({job=\"syslog\", mac=~\"$phone\"} |~ \"tftp to file failed|no config Provisioning Server|DURL<3[+]error\" [$__interval]))", + "queryType": "range", + "legendFormat": "Provisioning / download" + }, + { + "refId": "D", + "datasource": { "type": "loki", "uid": "${datasource}" }, + "expr": "sum(count_over_time({job=\"syslog\", mac=~\"$phone\"} |~ \"Timeout while contacting DNS servers\" [$__interval]))", + "queryType": "range", + "legendFormat": "SIP DNS timeout" } ], "options": { @@ -254,8 +275,8 @@ }, { "type": "timeseries", - "title": "Severity mix (emergency excluded — see notes)", - "description": "severity=emergency is filtered out: on Yealink it is the boot-time log-level table, not an incident.", + "title": "Diagnostic severity volume (not health)", + "description": "Shows what the firmware labeled by severity. Do not interpret the red area as failed phones: observed error-level traffic is dominated by benign internal diagnostics. Emergency is excluded because Yealink uses it for a boot-time log-level table.", "gridPos": { "h": 8, "w": 12, "x": 12, "y": 18 }, "id": 10, "datasource": { "type": "loki", "uid": "${datasource}" }, @@ -288,8 +309,8 @@ { "type": "logs", - "title": "Signals that matter — auth, provisioning, scheduler", - "description": "The three narrow patterns confirmed to carry real operational meaning. Read this first when a phone is reported faulty. Labels show both mac (stable identity) and source_ip (where it is on the network right now).", + "title": "Operational signals — investigate these first", + "description": "LDAP connectivity, internal database faults, provisioning/download failures, and SIP DNS timeouts. LDAP affects directory features but not calls; provisioning bursts during reboot are expected unless they persist. Labels show stable MAC and current source IP.", "gridPos": { "h": 10, "w": 24, "x": 0, "y": 26 }, "id": 11, "datasource": { "type": "loki", "uid": "${datasource}" }, @@ -297,7 +318,7 @@ { "refId": "A", "datasource": { "type": "loki", "uid": "${datasource}" }, - "expr": "{job=\"syslog\", mac=~\"$phone\"} |~ \"check passwd err|tftp to file failed|no config Provisioning Server|data_task schedule time out\"", + "expr": "{job=\"syslog\", mac=~\"$phone\"} |~ \"ldap_sasl_bind_s|Can't contact LDAP server|database disk image is malformed|Null Virtual Machine pointer|Invalid scalar query|tftp to file failed|no config Provisioning Server|DURL<3[+]error|Timeout while contacting DNS servers\"", "queryType": "range" } ], @@ -314,7 +335,8 @@ { "type": "logs", - "title": "All actionable errors (rtpcap noise floor removed)", + "title": "Raw firmware error severity (diagnostic only)", + "description": "For per-phone troubleshooting after the operational panels. These lines are not all actionable: Yealink marks substantial internal/boot chatter as error. Only the very high-volume RTP capture signature is removed to keep the panel usable.", "gridPos": { "h": 10, "w": 24, "x": 0, "y": 36 }, "id": 12, "datasource": { "type": "loki", "uid": "${datasource}" }, @@ -436,6 +458,6 @@ "timezone": "browser", "title": "Yealink IP phones — syslog health", "uid": "yealink-phones", - "version": 2, + "version": 3, "weekStart": "" }