From f438f3845c245c7b5fb3e9a7d65bcbad9addf50a Mon Sep 17 00:00:00 2001 From: pipistrello Date: Wed, 15 Jul 2026 19:55:09 +0300 Subject: [PATCH] Key Yealink phones by MAC, and stop pretending fleet-wide sums are thresholds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two changes driven by the fleet syslog rollout (~4 -> 106 senders) and the operator enabling Yealink's prepend-MAC on the autoprovisioned handsets. 1. Alloy: lift the MAC into a real `mac` label. UDP syslog offers only three possible identities: sender IP, RFC3164 HOSTNAME, message body. - source_ip is DHCP (observed lease 1200s) so it is NOT stable: a re-IP looks like a new device, and a recycled IP silently inherits another phone's history. - HOSTNAME is useless: Yealink puts its subsystem there (sua/GUI/cfg/sys), and after prepend-MAC the bracketed MAC lands in that slot and is rejected as a hostname, so the label is simply absent on new lines. So the body is the only place identity exists. A loki.process regex stage extracts it. Non-Yealink senders (the D-Link switches) don't match and pass through with no mac label. Cardinality is safe: mac is 1:1 with a device and does not multiply against source_ip. Verified on a disposable alloy+loki rig by replaying 406 REAL captured lines: 0 parse errors, both branches confirmed — Yealink lines get mac=80:5e:0c:b2:44:d3, the switch line passes through with no mac label. 2. Dashboard: identity by MAC, comparative panels, no invented thresholds. - Variable is now label_values(mac) with allValue ".+" (not ".*") so "All" matches only streams that HAVE a mac — excluding switches automatically. - Removed the absolute colour thresholds. They were derived from a ~4-hour sample of TWO atypically verbose handsets and were already wrong at 2 phones (actionable errors read 537 against a red line of 800; scheduler timeouts 193 against yellow at 150). They were fleet-wide SUMS, so they scale with phone count rather than with health. The fleet onboarded today, so no representative 24h baseline exists to replace them with — panels are comparative (worst offenders) until a week of real data exists. The one surviving threshold is "phones failing provisioning" >= 1, which is a qualitative fault rather than a magnitude. - Stats now count AFFECTED DEVICES, not fleet-wide line totals. - "Phones reporting" -> "Phones heard from", documented as NOT a health count: most of the fleet runs at Yealink default log level 3 (Error) and is silent unless something breaks. A silent phone is a healthy phone. - Registration/DHCP panels labelled level-6-only: they key on info/notice lines that level-3 phones never send, so an absent phone there means "quiet", not "unregistered". - Added "loudest phones by log volume" to surface handsets left at level 6 (two of them are ~80% of all volume). - refresh 1m -> 5m. Verified by firing all 14 panel queries CONCURRENTLY against a sink loaded with replayed real data: 14/14 return data, 0 rejected. Last time these were verified sequentially, which is what let the 429 bug ship. Note: data predating the MAC rollout has no mac label and won't appear. Co-Authored-By: Claude Opus 4.8 --- logging/alloy-config.alloy | 38 ++- .../grafana-dashboards/yealink-phones.json | 306 ++++++------------ 2 files changed, 141 insertions(+), 203 deletions(-) diff --git a/logging/alloy-config.alloy b/logging/alloy-config.alloy index 88c068a..ca0b928 100644 --- a/logging/alloy-config.alloy +++ b/logging/alloy-config.alloy @@ -35,6 +35,40 @@ loki.relabel "syslog" { // A listener can only parse one format; if a device ever sends genuine // rfc5424, it needs its own listener on a different port. +// Lifts the Yealink device MAC out of the message body into a real `mac` label. +// +// Why this is needed: UDP syslog gives us only three possible identities — the +// sender IP, the RFC3164 HOSTNAME field, and the message body. +// * source_ip is DHCP (observed lease time: 1200s), so it is NOT stable. A +// phone that re-IPs becomes a "new device", and one that inherits a +// recycled IP silently inherits another phone's history. +// * HOSTNAME is useless here: Yealink puts its internal *subsystem* there +// (sua, GUI, cfg, sys...). After enabling prepend-MAC the bracketed MAC +// occupies that slot and is rejected as a hostname, so the label is simply +// absent on new lines. +// So the body is the only place the device identity exists. Yealink's +// `static.syslog.prepend_mac_address.enable` puts it at the head of every line: +// [80:5e:0c:b2:44:d3] sua [824.1146]: FSM <6+info > [255] free nist ressource +// +// Non-Yealink senders (e.g. the D-Link switches) don't match the regex and are +// passed through untouched with no `mac` label — that is intentional. +// +// Cardinality is safe: mac is 1:1 with a device, so it does not multiply +// against source_ip (~100 phones -> ~100 values, and the pair is stable). +loki.process "extract_mac" { + forward_to = [loki.write.default.receiver] + + stage.regex { + expression = "^\\[(?P(?:[0-9a-fA-F]{2}:){5}[0-9a-fA-F]{2})\\]" + } + + stage.labels { + values = { + mac = "mac", + } + } +} + loki.source.syslog "network_devices_udp" { listener { address = "0.0.0.0:514" @@ -47,7 +81,7 @@ loki.source.syslog "network_devices_udp" { } relabel_rules = loki.relabel.syslog.rules - forward_to = [loki.write.default.receiver] + forward_to = [loki.process.extract_mac.receiver] } loki.source.syslog "network_devices_tcp" { @@ -62,7 +96,7 @@ loki.source.syslog "network_devices_tcp" { } relabel_rules = loki.relabel.syslog.rules - forward_to = [loki.write.default.receiver] + forward_to = [loki.process.extract_mac.receiver] } loki.write "default" { diff --git a/logging/grafana-dashboards/yealink-phones.json b/logging/grafana-dashboards/yealink-phones.json index a73544d..24da401 100644 --- a/logging/grafana-dashboards/yealink-phones.json +++ b/logging/grafana-dashboards/yealink-phones.json @@ -21,123 +21,95 @@ { "type": "text", "title": "How to read this dashboard", - "gridPos": { "h": 4, "w": 24, "x": 0, "y": 0 }, + "gridPos": { "h": 5, "w": 24, "x": 0, "y": 0 }, "id": 100, "options": { "mode": "markdown", - "content": "**Device identity is `source_ip`, not `hostname`.** Yealink phones put their internal *subsystem* (`sua`, `GUI`, `cfg`, `sys`, `ipp`…) in the syslog HOSTNAME field, so the `hostname` label cannot identify a device. Every panel here keys on `source_ip`.\n\n**Two things that look like alarms and are not.** `severity=emergency` is the phone printing its own log-level table at boot (`sys log :type=1,E=3,W=4,N=5,I=6,D=7`) — it is emitted at level 0 and means nothing; it is excluded here. `LSYS<3+error> rtpcap get len not enough` was **56% of all error lines** in the 24 h baseline — it is a noise floor and is excluded from every \"actionable\" panel below.\n\n**Panels filter to Yealink-format lines** via the `` module marker, so switches and routers sharing this Loki do not pollute the phone view." + "content": "**Identity is the phone's MAC**, lifted out of the message body into a `mac` label by Alloy (Yealink `static.syslog.prepend_mac_address.enable`). Do **not** use `source_ip` as identity — phones are on DHCP with a 1200 s lease, so a re-IP looks like a new device and a recycled IP would inherit another phone's history. Only devices sending a MAC appear here, so switches sharing this Loki are excluded automatically. *Data from before the MAC rollout has no `mac` label and will not show.*\n\n**A silent phone is a healthy phone, not a missing one.** Most of the fleet runs at Yealink's default log level **3 (Error)** and only speaks when something is wrong. A couple of handsets are at level **6 (Informational)** and produce ~80% of all volume. These panels count *phones heard from*, never *phones alive* — and the registration/DHCP panels at the bottom can only ever populate for level-6 phones.\n\n**Two known noise sources are filtered out:** `severity=emergency` is Yealink printing its own log-level table at boot (`E=3,W=4,N=5,I=6,D=7`), not an incident; and `LSYS<3+error> rtpcap get len not enough` was 56% of all error lines in baseline. Neither means anything.\n\n**No colour thresholds on the counts — deliberately.** The fleet onboarded 2026-07-15, so there is no representative 24 h baseline to derive them from, and fleet-wide sums scale with phone count rather than with health. The panels are *comparative* (worst offenders) instead. Revisit once a full week of fleet data exists." } }, { "type": "stat", - "title": "Phones reporting", - "description": "Distinct source_ip emitting Yealink-format syslog in the selected range.", - "gridPos": { "h": 4, "w": 4, "x": 0, "y": 4 }, + "title": "Phones heard from", + "description": "Distinct MACs that sent at least one line in range. NOT a health count — level-3 phones stay silent unless something breaks, so this is expected to sit well below the fleet size.", + "gridPos": { "h": 4, "w": 4, "x": 0, "y": 5 }, "id": 1, "datasource": { "type": "loki", "uid": "${datasource}" }, "targets": [ { "refId": "A", "datasource": { "type": "loki", "uid": "${datasource}" }, - "expr": "count(count by (source_ip) (count_over_time({job=\"syslog\", source_ip=~\"$phone\"} |~ \"<[0-7][+]\" [$__range])))", + "expr": "count(count by (mac) (count_over_time({job=\"syslog\", mac=~\"$phone\"} [$__range])))", "queryType": "instant" } ], "options": { "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, - "colorMode": "value", + "colorMode": "none", "graphMode": "none", - "textMode": "auto", - "justifyMode": "auto" + "textMode": "auto" }, - "fieldConfig": { - "defaults": { - "unit": "short", - "color": { "mode": "fixed", "fixedColor": "text" }, - "mappings": [] - }, - "overrides": [] - } + "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "fixed", "fixedColor": "text" } }, "overrides": [] } }, { "type": "stat", "title": "Log lines", - "description": "Total Yealink-format syslog lines in range.", - "gridPos": { "h": 4, "w": 4, "x": 4, "y": 4 }, + "description": "Total lines from MAC-identified phones in range.", + "gridPos": { "h": 4, "w": 4, "x": 4, "y": 5 }, "id": 2, "datasource": { "type": "loki", "uid": "${datasource}" }, "targets": [ { "refId": "A", "datasource": { "type": "loki", "uid": "${datasource}" }, - "expr": "sum(count_over_time({job=\"syslog\", source_ip=~\"$phone\"} |~ \"<[0-7][+]\" [$__range]))", + "expr": "sum(count_over_time({job=\"syslog\", mac=~\"$phone\"} [$__range]))", "queryType": "instant" } ], "options": { "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, - "colorMode": "value", + "colorMode": "none", "graphMode": "none", "textMode": "auto" }, - "fieldConfig": { - "defaults": { - "unit": "short", - "color": { "mode": "fixed", "fixedColor": "text" } - }, - "overrides": [] - } + "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "fixed", "fixedColor": "text" } }, "overrides": [] } }, { "type": "stat", - "title": "Actionable errors", - "description": "severity=error, excluding the 'rtpcap get len not enough' noise floor (56% of raw errors in baseline).", - "gridPos": { "h": 4, "w": 4, "x": 8, "y": 4 }, + "title": "Phones with errors", + "description": "How many distinct handsets logged at least one actionable error (rtpcap noise floor excluded). A count of affected devices — unlike a fleet-wide error total, this does not inflate as phones are added.", + "gridPos": { "h": 4, "w": 4, "x": 8, "y": 5 }, "id": 3, "datasource": { "type": "loki", "uid": "${datasource}" }, "targets": [ { "refId": "A", "datasource": { "type": "loki", "uid": "${datasource}" }, - "expr": "sum(count_over_time({job=\"syslog\", source_ip=~\"$phone\", severity=\"error\"} !~ \"rtpcap get len not enough\" [$__range]))", + "expr": "count(count by (mac) (count_over_time({job=\"syslog\", mac=~\"$phone\", severity=\"error\"} !~ \"rtpcap get len not enough\" [$__range])))", "queryType": "instant" } ], "options": { "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, - "colorMode": "value", + "colorMode": "none", "graphMode": "none", "textMode": "auto" }, - "fieldConfig": { - "defaults": { - "unit": "short", - "color": { "mode": "thresholds" }, - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 200 }, - { "color": "red", "value": 800 } - ] - } - }, - "overrides": [] - } + "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "fixed", "fixedColor": "text" } }, "overrides": [] } }, { "type": "stat", - "title": "Auth failures", - "description": "CUIT 'check passwd err' — phone web/UI credential rejections. Baseline: 21 / 24 h.", - "gridPos": { "h": 4, "w": 4, "x": 12, "y": 4 }, + "title": "Phones failing provisioning", + "description": "Handsets hitting 'tftp to file failed' or 'no config Provisioning Server'. Any non-zero value is worth a look regardless of fleet size — this is a qualitative fault, not a magnitude, which is why it carries a threshold when the count panels deliberately do not.", + "gridPos": { "h": 4, "w": 4, "x": 12, "y": 5 }, "id": 4, "datasource": { "type": "loki", "uid": "${datasource}" }, "targets": [ { "refId": "A", "datasource": { "type": "loki", "uid": "${datasource}" }, - "expr": "sum(count_over_time({job=\"syslog\", source_ip=~\"$phone\"} |~ \"check passwd err\" [$__range]))", + "expr": "count(count by (mac) (count_over_time({job=\"syslog\", mac=~\"$phone\"} |~ \"tftp to file failed|no config Provisioning Server\" [$__range])))", "queryType": "instant" } ], @@ -151,179 +123,119 @@ "defaults": { "unit": "short", "color": { "mode": "thresholds" }, - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 25 }, - { "color": "red", "value": 100 } - ] - } + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 1 }] } }, "overrides": [] } }, { "type": "stat", - "title": "Provisioning failures", - "description": "ATP 'tftp to file failed' and SYS 'no config Provisioning Server'. Any non-zero value is worth a look — baseline was 3 / 24 h.", - "gridPos": { "h": 4, "w": 4, "x": 16, "y": 4 }, + "title": "Phones with auth failures", + "description": "Handsets logging CUIT 'check passwd err' — a credential rejection on the phone's own web/UI. Could be a mis-provisioned password or someone probing; read the log panel below rather than treating the number as a score.", + "gridPos": { "h": 4, "w": 4, "x": 16, "y": 5 }, "id": 5, "datasource": { "type": "loki", "uid": "${datasource}" }, "targets": [ { "refId": "A", "datasource": { "type": "loki", "uid": "${datasource}" }, - "expr": "sum(count_over_time({job=\"syslog\", source_ip=~\"$phone\"} |~ \"tftp to file failed|no config Provisioning Server\" [$__range]))", + "expr": "count(count by (mac) (count_over_time({job=\"syslog\", mac=~\"$phone\"} |~ \"check passwd err\" [$__range])))", "queryType": "instant" } ], "options": { "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, - "colorMode": "value", + "colorMode": "none", "graphMode": "none", "textMode": "auto" }, - "fieldConfig": { - "defaults": { - "unit": "short", - "color": { "mode": "thresholds" }, - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 1 }, - { "color": "red", "value": 20 } - ] - } - }, - "overrides": [] - } + "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "fixed", "fixedColor": "text" } }, "overrides": [] } }, { "type": "stat", - "title": "Scheduling timeouts", - "description": "DEV 'data_task schedule time out' — the phone's internal task scheduler overran its 30 s threshold. A rising count means the handset is struggling, and is the closest thing here to a 'phone is unwell' signal. Baseline: 117 / 24 h.", - "gridPos": { "h": 4, "w": 4, "x": 20, "y": 4 }, + "title": "Phones with scheduler timeouts", + "description": "Handsets logging DEV 'data_task schedule time out' — the phone's internal scheduler overran its 30 s threshold. The best available proxy for 'this handset is struggling'.", + "gridPos": { "h": 4, "w": 4, "x": 20, "y": 5 }, "id": 6, "datasource": { "type": "loki", "uid": "${datasource}" }, "targets": [ { "refId": "A", "datasource": { "type": "loki", "uid": "${datasource}" }, - "expr": "sum(count_over_time({job=\"syslog\", source_ip=~\"$phone\"} |~ \"data_task schedule time out\" [$__range]))", + "expr": "count(count by (mac) (count_over_time({job=\"syslog\", mac=~\"$phone\"} |~ \"data_task schedule time out\" [$__range])))", "queryType": "instant" } ], "options": { "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, - "colorMode": "value", + "colorMode": "none", "graphMode": "none", "textMode": "auto" }, - "fieldConfig": { - "defaults": { - "unit": "short", - "color": { "mode": "thresholds" }, - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 150 }, - { "color": "red", "value": 500 } - ] - } - }, - "overrides": [] - } + "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "fixed", "fixedColor": "text" } }, "overrides": [] } }, { - "type": "timeseries", - "title": "Log rate by phone", - "description": "Lines/sec per device. A phone that goes flat has stopped sending syslog — that is either a dead handset, a reboot, or syslog switched off on the device (it silently defaults to disabled).", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, + "type": "bargauge", + "title": "Worst phones by actionable errors", + "description": "Comparative, not absolute — find the outlier rather than judge against a made-up number. Keyed on MAC, so a phone that changes IP stays the same bar.", + "gridPos": { "h": 9, "w": 12, "x": 0, "y": 9 }, "id": 7, "datasource": { "type": "loki", "uid": "${datasource}" }, "targets": [ { "refId": "A", "datasource": { "type": "loki", "uid": "${datasource}" }, - "expr": "sum by (source_ip) (rate({job=\"syslog\", source_ip=~\"$phone\"} |~ \"<[0-7][+]\" [$__interval]))", - "queryType": "range", - "legendFormat": "{{source_ip}}" + "expr": "topk(10, sum by (mac) (count_over_time({job=\"syslog\", mac=~\"$phone\", severity=\"error\"} !~ \"rtpcap get len not enough\" [$__range])))", + "queryType": "instant", + "legendFormat": "{{mac}}" } ], "options": { - "legend": { "displayMode": "table", "placement": "right", "showLegend": true, "calcs": ["mean", "max"] }, - "tooltip": { "mode": "multi", "sort": "desc" } + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "displayMode": "gradient", + "orientation": "horizontal", + "showUnfilled": true }, - "fieldConfig": { - "defaults": { - "unit": "logs/s (calculated)", - "custom": { - "drawStyle": "line", - "lineWidth": 1, - "fillOpacity": 10, - "showPoints": "never", - "spanNulls": false - } - }, - "overrides": [] - } + "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "continuous-GrYlRd" } }, "overrides": [] } }, { - "type": "timeseries", - "title": "Severity mix (emergency excluded — see notes)", - "description": "severity=emergency is deliberately filtered out: on Yealink it is the boot-time log-level table, not an incident.", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 }, + "type": "bargauge", + "title": "Loudest phones by log volume", + "description": "Identifies handsets left at log level 6 (Informational). Yealink's default is level 3 (Error); a phone near the top here is verbose by configuration, not necessarily unhealthy — but it dominates disk and query cost.", + "gridPos": { "h": 9, "w": 12, "x": 12, "y": 9 }, "id": 8, "datasource": { "type": "loki", "uid": "${datasource}" }, "targets": [ { "refId": "A", "datasource": { "type": "loki", "uid": "${datasource}" }, - "expr": "sum by (severity) (count_over_time({job=\"syslog\", source_ip=~\"$phone\", severity!=\"emergency\"} |~ \"<[0-7][+]\" [$__interval]))", - "queryType": "range", - "legendFormat": "{{severity}}" + "expr": "topk(10, sum by (mac) (count_over_time({job=\"syslog\", mac=~\"$phone\"} [$__range])))", + "queryType": "instant", + "legendFormat": "{{mac}}" } ], "options": { - "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, - "tooltip": { "mode": "multi", "sort": "desc" } + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "displayMode": "gradient", + "orientation": "horizontal", + "showUnfilled": true }, - "fieldConfig": { - "defaults": { - "unit": "short", - "custom": { - "drawStyle": "bars", - "lineWidth": 0, - "fillOpacity": 70, - "stacking": { "mode": "normal", "group": "A" }, - "showPoints": "never" - } - }, - "overrides": [ - { "matcher": { "id": "byName", "options": "error" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } }] }, - { "matcher": { "id": "byName", "options": "warning" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "orange" } }] }, - { "matcher": { "id": "byName", "options": "notice" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "blue" } }] }, - { "matcher": { "id": "byName", "options": "informational" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "green" } }] } - ] - } + "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "continuous-BlPu" } }, "overrides": [] } }, { "type": "timeseries", "title": "Actionable errors by Yealink module", - "description": "Module tag parsed out of the message body (it is not a label). CFG = config attribute lookups, DEV = scheduler, CUIT = auth, ATP = provisioning/transfer, WEB = web UI.", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 }, + "description": "Module tag parsed from the message body (it is not a label). CFG = config attribute lookups, DEV = scheduler, CUIT = auth, ATP = provisioning/transfer, WEB = web UI, LSYS = RTP capture.", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 18 }, "id": 9, "datasource": { "type": "loki", "uid": "${datasource}" }, "targets": [ { "refId": "A", "datasource": { "type": "loki", "uid": "${datasource}" }, - "expr": "sum by (module) (count_over_time({job=\"syslog\", source_ip=~\"$phone\", severity=\"error\"} !~ \"rtpcap get len not enough\" | regexp \"(?P[A-Z]{2,5}) *<3[+]error\" [$__interval]))", + "expr": "sum by (module) (count_over_time({job=\"syslog\", mac=~\"$phone\", severity=\"error\"} !~ \"rtpcap get len not enough\" | regexp \"(?P[A-Z]{2,5}) *<3[+]error\" [$__interval]))", "queryType": "range", "legendFormat": "{{module}}" } @@ -335,60 +247,57 @@ "fieldConfig": { "defaults": { "unit": "short", - "custom": { - "drawStyle": "bars", - "lineWidth": 0, - "fillOpacity": 80, - "stacking": { "mode": "normal", "group": "A" }, - "showPoints": "never" - } + "custom": { "drawStyle": "bars", "lineWidth": 0, "fillOpacity": 80, "stacking": { "mode": "normal", "group": "A" }, "showPoints": "never" } }, "overrides": [] } }, { - "type": "bargauge", - "title": "Actionable errors by phone", - "description": "Which handset is actually generating the errors, noise floor removed.", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 }, + "type": "timeseries", + "title": "Severity mix (emergency excluded — see notes)", + "description": "severity=emergency is filtered out: on Yealink it is the boot-time log-level table, not an incident.", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 18 }, "id": 10, "datasource": { "type": "loki", "uid": "${datasource}" }, "targets": [ { "refId": "A", "datasource": { "type": "loki", "uid": "${datasource}" }, - "expr": "sum by (source_ip) (count_over_time({job=\"syslog\", source_ip=~\"$phone\", severity=\"error\"} !~ \"rtpcap get len not enough\" [$__range]))", - "queryType": "instant", - "legendFormat": "{{source_ip}}" + "expr": "sum by (severity) (count_over_time({job=\"syslog\", mac=~\"$phone\", severity!=\"emergency\"} [$__interval]))", + "queryType": "range", + "legendFormat": "{{severity}}" } ], "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, - "displayMode": "gradient", - "orientation": "horizontal", - "showUnfilled": true + "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "desc" } }, "fieldConfig": { "defaults": { "unit": "short", - "color": { "mode": "continuous-GrYlRd" } + "custom": { "drawStyle": "bars", "lineWidth": 0, "fillOpacity": 70, "stacking": { "mode": "normal", "group": "A" }, "showPoints": "never" } }, - "overrides": [] + "overrides": [ + { "matcher": { "id": "byName", "options": "error" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } }] }, + { "matcher": { "id": "byName", "options": "warning" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "orange" } }] }, + { "matcher": { "id": "byName", "options": "notice" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "blue" } }] }, + { "matcher": { "id": "byName", "options": "informational" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "green" } }] } + ] } }, { "type": "logs", "title": "Signals that matter — auth, provisioning, scheduler", - "description": "The three narrow patterns confirmed to carry real operational meaning on these handsets. This is the panel to read first when a phone is reported faulty.", - "gridPos": { "h": 10, "w": 24, "x": 0, "y": 24 }, + "description": "The three narrow patterns confirmed to carry real operational meaning. Read this first when a phone is reported faulty. Labels show both mac (stable identity) and source_ip (where it is on the network right now).", + "gridPos": { "h": 10, "w": 24, "x": 0, "y": 26 }, "id": 11, "datasource": { "type": "loki", "uid": "${datasource}" }, "targets": [ { "refId": "A", "datasource": { "type": "loki", "uid": "${datasource}" }, - "expr": "{job=\"syslog\", source_ip=~\"$phone\"} |~ \"check passwd err|tftp to file failed|no config Provisioning Server|data_task schedule time out\"", + "expr": "{job=\"syslog\", mac=~\"$phone\"} |~ \"check passwd err|tftp to file failed|no config Provisioning Server|data_task schedule time out\"", "queryType": "range" } ], @@ -397,7 +306,6 @@ "showLabels": true, "showCommonLabels": false, "wrapLogMessage": true, - "prettifyLogMessage": false, "enableLogDetails": true, "dedupStrategy": "none", "sortOrder": "Descending" @@ -407,14 +315,14 @@ { "type": "logs", "title": "All actionable errors (rtpcap noise floor removed)", - "gridPos": { "h": 10, "w": 24, "x": 0, "y": 34 }, + "gridPos": { "h": 10, "w": 24, "x": 0, "y": 36 }, "id": 12, "datasource": { "type": "loki", "uid": "${datasource}" }, "targets": [ { "refId": "A", "datasource": { "type": "loki", "uid": "${datasource}" }, - "expr": "{job=\"syslog\", source_ip=~\"$phone\", severity=\"error\"} !~ \"rtpcap get len not enough\"", + "expr": "{job=\"syslog\", mac=~\"$phone\", severity=\"error\"} !~ \"rtpcap get len not enough\"", "queryType": "range" } ], @@ -431,18 +339,18 @@ { "type": "timeseries", - "title": "SIP registration refresh", - "description": "REG 'Register: update server' events. Registration period observed at 120 s, so a healthy phone should show a steady beat here; gaps mean the handset lost its registration to the PBX (192.168.0.8).", - "gridPos": { "h": 7, "w": 12, "x": 0, "y": 44 }, + "title": "SIP registration refresh (level-6 phones only)", + "description": "REG 'Register: update server' fires at info level, so ONLY handsets left at log level 6 appear here. A gap for such a phone means it lost registration to the PBX (192.168.0.8); an absent phone means it is at level 3, NOT that it is unregistered.", + "gridPos": { "h": 7, "w": 12, "x": 0, "y": 46 }, "id": 13, "datasource": { "type": "loki", "uid": "${datasource}" }, "targets": [ { "refId": "A", "datasource": { "type": "loki", "uid": "${datasource}" }, - "expr": "sum by (source_ip) (count_over_time({job=\"syslog\", source_ip=~\"$phone\"} |~ \"Register: update server\" [$__interval]))", + "expr": "sum by (mac) (count_over_time({job=\"syslog\", mac=~\"$phone\"} |~ \"Register: update server\" [$__interval]))", "queryType": "range", - "legendFormat": "{{source_ip}}" + "legendFormat": "{{mac}}" } ], "options": { @@ -452,35 +360,30 @@ "fieldConfig": { "defaults": { "unit": "short", - "custom": { - "drawStyle": "points", - "pointSize": 5, - "lineWidth": 1, - "fillOpacity": 0, - "showPoints": "always" - } + "custom": { "drawStyle": "points", "pointSize": 5, "lineWidth": 1, "fillOpacity": 0, "showPoints": "always" } }, "overrides": [] } }, { "type": "logs", - "title": "DHCP lease events", - "description": "Lease acquisition/renewal as the phone sees it. Useful when a handset is 'connected but dead' — cross-reference against the router's DHCP log.", - "gridPos": { "h": 7, "w": 12, "x": 12, "y": 44 }, + "title": "DHCP lease events (level-6 phones only)", + "description": "Lease acquisition/renewal as the phone sees it — notice level, so the same caveat as the registration panel. Useful when a handset is 'connected but dead'. Observed lease time is 1200 s, which is exactly why source_ip is not a safe identity.", + "gridPos": { "h": 7, "w": 12, "x": 12, "y": 46 }, "id": 14, "datasource": { "type": "loki", "uid": "${datasource}" }, "targets": [ { "refId": "A", "datasource": { "type": "loki", "uid": "${datasource}" }, - "expr": "{job=\"syslog\", source_ip=~\"$phone\"} |~ \"lease of .* obtained|lease renew\"", + "expr": "{job=\"syslog\", mac=~\"$phone\"} |~ \"lease of .* obtained|lease renew\"", "queryType": "range" } ], "options": { "showTime": true, - "showLabels": false, + "showLabels": true, + "showCommonLabels": false, "wrapLogMessage": true, "enableLogDetails": true, "dedupStrategy": "none", @@ -488,7 +391,7 @@ } } ], - "refresh": "1m", + "refresh": "5m", "schemaVersion": 39, "tags": ["yealink", "phones", "syslog", "02-tekhnohim"], "templating": { @@ -506,12 +409,13 @@ }, { "name": "phone", - "label": "Phone (source_ip)", + "label": "Phone (MAC)", + "description": "Device identity. allValue is '.+' (not '.*') so that 'All' matches only streams that HAVE a mac label — which excludes switches and other non-Yealink senders sharing this Loki.", "type": "query", "datasource": { "type": "loki", "uid": "${datasource}" }, - "definition": "label_values({job=\"syslog\"}, source_ip)", + "definition": "label_values({job=\"syslog\"}, mac)", "query": { - "label": "source_ip", + "label": "mac", "refId": "LokiVariableQueryEditor-VariableQuery", "stream": "{job=\"syslog\"}", "type": 1 @@ -519,7 +423,7 @@ "current": { "selected": true, "text": ["All"], "value": ["$__all"] }, "includeAll": true, "multi": true, - "allValue": ".*", + "allValue": ".+", "hide": 0, "refresh": 2, "sort": 1, @@ -532,6 +436,6 @@ "timezone": "browser", "title": "Yealink IP phones — syslog health", "uid": "yealink-phones", - "version": 1, + "version": 2, "weekStart": "" }