Compare commits
38
Commits
e511b19ac5
..
master
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3bd63ec710 | ||
|
|
673d747abd | ||
|
|
d40941dbde | ||
|
|
229fddfc89 | ||
|
|
8f95385b89 | ||
|
|
ebf94e668b | ||
|
|
07c4495a03 | ||
|
|
36376e60a1 | ||
|
|
155db049b0 | ||
|
|
46bb775939 | ||
|
|
7ebe9f2a7e | ||
|
|
0f8620c405 | ||
|
|
a4aa0e90fe | ||
|
|
1a42f89c11 | ||
|
|
43be2d8bb0 | ||
|
|
1496e07d19 | ||
|
|
f438f3845c | ||
|
|
bb4c92d5e5 | ||
|
|
13b1887a16 | ||
|
|
9809cddbdf | ||
|
|
fb3ac50522 | ||
|
|
4c1033b460 | ||
|
|
22dfb732d3 | ||
|
|
c3589732f2 | ||
|
|
951135658a | ||
|
|
91bedb35eb | ||
|
|
e957700a5b | ||
|
|
2d37c51730 | ||
|
|
127aad886a | ||
|
|
c3bb96be45 | ||
|
|
2892f25229 | ||
|
|
793c6dbc5c | ||
|
|
5c9f91cd3d | ||
|
|
33ad5ddb0b | ||
|
|
9c3475edae | ||
|
|
9bee0f99d0 | ||
|
|
25089cb668 | ||
|
|
8dd2ef0151 |
@@ -6,15 +6,4 @@ services:
|
|||||||
container_name: grafana
|
container_name: grafana
|
||||||
ports:
|
ports:
|
||||||
- "3000:3000"
|
- "3000:3000"
|
||||||
environment:
|
|
||||||
- GF_SECURITY_ADMIN_PASSWORD={$GRAFANA_ADMIN_PASSWORD}
|
|
||||||
volumes:
|
|
||||||
- /mnt/containers/grafana/container-data:/var/lib/grafana
|
|
||||||
networks:
|
|
||||||
- reverseproxy-nw
|
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
|
|
||||||
|
|
||||||
networks:
|
|
||||||
reverseproxy-nw:
|
|
||||||
external: true
|
|
||||||
|
|||||||
@@ -0,0 +1,106 @@
|
|||||||
|
// Receives syslog from routers, switches, and Yealink phones on this host's
|
||||||
|
// LAN interface, tags each message with useful labels, and ships it to the
|
||||||
|
// local Loki instance.
|
||||||
|
//
|
||||||
|
// Point network devices at this host's IP, port 514/udp (or 1514/tcp for
|
||||||
|
// devices that only speak TCP framing).
|
||||||
|
|
||||||
|
loki.relabel "syslog" {
|
||||||
|
forward_to = []
|
||||||
|
|
||||||
|
rule {
|
||||||
|
source_labels = ["__syslog_message_hostname"]
|
||||||
|
target_label = "hostname"
|
||||||
|
}
|
||||||
|
|
||||||
|
rule {
|
||||||
|
source_labels = ["__syslog_message_severity"]
|
||||||
|
target_label = "severity"
|
||||||
|
}
|
||||||
|
|
||||||
|
rule {
|
||||||
|
source_labels = ["__syslog_message_facility"]
|
||||||
|
target_label = "facility"
|
||||||
|
}
|
||||||
|
|
||||||
|
rule {
|
||||||
|
source_labels = ["__syslog_connection_ip_address"]
|
||||||
|
target_label = "source_ip"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// syslog_format is "rfc3164" (legacy BSD syslog, no version field) rather
|
||||||
|
// than the component's rfc5424 default — confirmed by packet capture that
|
||||||
|
// both the Yealink phones and RouterOS routers send rfc3164 in practice.
|
||||||
|
// A listener can only parse one format; if a device ever sends genuine
|
||||||
|
// rfc5424, it needs its own listener on a different port.
|
||||||
|
|
||||||
|
// Lifts the Yealink device MAC out of the message body into a real `mac` label.
|
||||||
|
//
|
||||||
|
// Why this is needed: UDP syslog gives us only three possible identities — the
|
||||||
|
// sender IP, the RFC3164 HOSTNAME field, and the message body.
|
||||||
|
// * source_ip is DHCP (observed lease time: 1200s), so it is NOT stable. A
|
||||||
|
// phone that re-IPs becomes a "new device", and one that inherits a
|
||||||
|
// recycled IP silently inherits another phone's history.
|
||||||
|
// * HOSTNAME is useless here: Yealink puts its internal *subsystem* there
|
||||||
|
// (sua, GUI, cfg, sys...). After enabling prepend-MAC the bracketed MAC
|
||||||
|
// occupies that slot and is rejected as a hostname, so the label is simply
|
||||||
|
// absent on new lines.
|
||||||
|
// So the body is the only place the device identity exists. Yealink's
|
||||||
|
// `static.syslog.prepend_mac_address.enable` puts it at the head of every line:
|
||||||
|
// [80:5e:0c:b2:44:d3] sua [824.1146]: FSM <6+info > [255] free nist ressource
|
||||||
|
//
|
||||||
|
// Non-Yealink senders (e.g. the D-Link switches) don't match the regex and are
|
||||||
|
// passed through untouched with no `mac` label — that is intentional.
|
||||||
|
//
|
||||||
|
// Cardinality is safe: mac is 1:1 with a device, so it does not multiply
|
||||||
|
// against source_ip (~100 phones -> ~100 values, and the pair is stable).
|
||||||
|
loki.process "extract_mac" {
|
||||||
|
forward_to = [loki.write.default.receiver]
|
||||||
|
|
||||||
|
stage.regex {
|
||||||
|
expression = "^\\[(?P<mac>(?:[0-9a-fA-F]{2}:){5}[0-9a-fA-F]{2})\\]"
|
||||||
|
}
|
||||||
|
|
||||||
|
stage.labels {
|
||||||
|
values = {
|
||||||
|
mac = "mac",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
loki.source.syslog "network_devices_udp" {
|
||||||
|
listener {
|
||||||
|
address = "0.0.0.0:514"
|
||||||
|
protocol = "udp"
|
||||||
|
syslog_format = "rfc3164"
|
||||||
|
labels = {
|
||||||
|
job = "syslog",
|
||||||
|
transport = "udp",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
relabel_rules = loki.relabel.syslog.rules
|
||||||
|
forward_to = [loki.process.extract_mac.receiver]
|
||||||
|
}
|
||||||
|
|
||||||
|
loki.source.syslog "network_devices_tcp" {
|
||||||
|
listener {
|
||||||
|
address = "0.0.0.0:1514"
|
||||||
|
protocol = "tcp"
|
||||||
|
syslog_format = "rfc3164"
|
||||||
|
labels = {
|
||||||
|
job = "syslog",
|
||||||
|
transport = "tcp",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
relabel_rules = loki.relabel.syslog.rules
|
||||||
|
forward_to = [loki.process.extract_mac.receiver]
|
||||||
|
}
|
||||||
|
|
||||||
|
loki.write "default" {
|
||||||
|
endpoint {
|
||||||
|
url = "http://loki:3100/loki/api/v1/push"
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,71 @@
|
|||||||
|
version: '3.8'
|
||||||
|
|
||||||
|
services:
|
||||||
|
loki:
|
||||||
|
# Keep a fixed release for reproducible Portainer redeploys. The 2.9.8 ->
|
||||||
|
# 3.7.2 upgrade was validated against a copy of this stack's TSDB v13 data.
|
||||||
|
image: grafana/loki:3.7.2
|
||||||
|
container_name: loki
|
||||||
|
command: -config.file=/etc/loki/config.yaml
|
||||||
|
restart: unless-stopped
|
||||||
|
volumes:
|
||||||
|
- /mnt/containers/logging/loki-config/config.yaml:/etc/loki/config.yaml:Z
|
||||||
|
- /mnt/containers/logging/loki-data:/loki:Z
|
||||||
|
networks:
|
||||||
|
- logging-nw
|
||||||
|
|
||||||
|
alloy:
|
||||||
|
# Keep a fixed release for reproducible Portainer redeploys. The current
|
||||||
|
# RFC3164 config and an end-to-end Alloy -> Loki probe passed on v1.16.1.
|
||||||
|
image: grafana/alloy:v1.16.1
|
||||||
|
container_name: alloy
|
||||||
|
command:
|
||||||
|
- run
|
||||||
|
- --server.http.listen-addr=0.0.0.0:12345
|
||||||
|
- --storage.path=/var/lib/alloy/data
|
||||||
|
- /etc/alloy/config.alloy
|
||||||
|
restart: unless-stopped
|
||||||
|
depends_on:
|
||||||
|
- loki
|
||||||
|
volumes:
|
||||||
|
- /mnt/containers/logging/alloy-config/config.alloy:/etc/alloy/config.alloy:Z
|
||||||
|
- /mnt/containers/logging/alloy-data:/var/lib/alloy/data:Z
|
||||||
|
ports:
|
||||||
|
# syslog intake for routers/switches/phones (UDP, RFC3164/5424)
|
||||||
|
- '514:514/udp'
|
||||||
|
# optional TCP syslog for devices that don't do UDP
|
||||||
|
- '1514:1514/tcp'
|
||||||
|
networks:
|
||||||
|
- logging-nw
|
||||||
|
|
||||||
|
grafana:
|
||||||
|
# Keep a fixed release for reproducible Portainer redeploys. Grafana 13.1.0
|
||||||
|
# is supported by the host's Fedora 44 / kernel 7.1 runtime; the old 10.4.2
|
||||||
|
# pin was only required by the retired Fedora 35 / kernel 5.16 host.
|
||||||
|
image: grafana/grafana:13.1.0
|
||||||
|
container_name: grafana
|
||||||
|
restart: unless-stopped
|
||||||
|
depends_on:
|
||||||
|
- loki
|
||||||
|
environment:
|
||||||
|
- GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_ADMIN_PASSWORD}
|
||||||
|
- GF_USERS_ALLOW_SIGN_UP=false
|
||||||
|
# Install the Zabbix app before Grafana starts so its provisioned
|
||||||
|
# datasource and dashboards are available immediately after redeploy.
|
||||||
|
- GF_PLUGINS_PREINSTALL_SYNC=alexanderzobnin-zabbix-app@6.5.0
|
||||||
|
volumes:
|
||||||
|
- /mnt/containers/logging/grafana-data:/var/lib/grafana:Z
|
||||||
|
- /mnt/containers/logging/grafana-config/provisioning/datasources:/etc/grafana/provisioning/datasources:Z
|
||||||
|
# Dashboard provisioning: the provider config below points at the dashboards
|
||||||
|
# dir; both are file-provisioned, so UI edits are not persisted — edit the
|
||||||
|
# JSON in git and redeploy.
|
||||||
|
- /mnt/containers/logging/grafana-config/provisioning/dashboards:/etc/grafana/provisioning/dashboards:Z
|
||||||
|
- /mnt/containers/logging/grafana-config/dashboards:/var/lib/grafana/dashboards:Z
|
||||||
|
networks:
|
||||||
|
- logging-nw
|
||||||
|
- reverseproxy-nw
|
||||||
|
|
||||||
|
networks:
|
||||||
|
logging-nw:
|
||||||
|
reverseproxy-nw:
|
||||||
|
external: true
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,463 @@
|
|||||||
|
{
|
||||||
|
"annotations": {
|
||||||
|
"list": [
|
||||||
|
{
|
||||||
|
"builtIn": 1,
|
||||||
|
"datasource": { "type": "grafana", "uid": "-- Grafana --" },
|
||||||
|
"enable": true,
|
||||||
|
"hide": true,
|
||||||
|
"iconColor": "rgba(0, 211, 255, 1)",
|
||||||
|
"name": "Annotations & Alerts",
|
||||||
|
"type": "dashboard"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"editable": true,
|
||||||
|
"fiscalYearStartMonth": 0,
|
||||||
|
"graphTooltip": 1,
|
||||||
|
"links": [],
|
||||||
|
"liveNow": false,
|
||||||
|
"panels": [
|
||||||
|
{
|
||||||
|
"type": "text",
|
||||||
|
"title": "How to read this dashboard",
|
||||||
|
"gridPos": { "h": 5, "w": 24, "x": 0, "y": 0 },
|
||||||
|
"id": 100,
|
||||||
|
"options": {
|
||||||
|
"mode": "markdown",
|
||||||
|
"content": "**Identity is the phone's MAC**, lifted out of the message body into a `mac` label by Alloy (Yealink `static.syslog.prepend_mac_address.enable`). Do **not** use `source_ip` as identity — phones are on DHCP with a 1200 s lease, so a re-IP looks like a new device and a recycled IP would inherit another phone's history. Only devices sending a MAC appear here, so switches sharing this Loki are excluded automatically. *Data from before the MAC rollout has no `mac` label and will not show.*\n\n**Yealink's `error` severity is diagnostic verbosity, not a fleet health verdict.** Firmware-internal messages such as missing optional DHCP attributes, GUI event targets, RTP capture reads, scheduler overruns near the built-in 30 s threshold, kernel register dumps, and provisioning-engine state account for most red lines. Use the narrow **Operational signals** panels first; use raw severity only when investigating one handset.\n\n**The top status cards are intentionally recent.** LDAP and internal-database cards cover the last 15 minutes; provisioning covers the last hour. This prevents a completed fleet rollout or reboot from looking like a current outage for the rest of a 24 h dashboard range. LDAP failures affect directory/search and caller-name lookup, not SIP calling itself.\n\n**A silent phone is not proven healthy or offline.** Most of the fleet runs at log level 3 and only sends selected errors. Registration and DHCP evidence at the bottom is available only from verbose level-6 phones. `severity=emergency` is also excluded from the severity chart because Yealink uses it while printing its boot-time log-level table, not to report an emergency."
|
||||||
|
}
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
"type": "stat",
|
||||||
|
"title": "Phones heard from",
|
||||||
|
"description": "Distinct MACs that sent at least one line in range. NOT a health count — level-3 phones stay silent unless something breaks, so this is expected to sit well below the fleet size.",
|
||||||
|
"gridPos": { "h": 4, "w": 4, "x": 0, "y": 5 },
|
||||||
|
"id": 1,
|
||||||
|
"datasource": { "type": "loki", "uid": "${datasource}" },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"refId": "A",
|
||||||
|
"datasource": { "type": "loki", "uid": "${datasource}" },
|
||||||
|
"expr": "count(count by (mac) (count_over_time({job=\"syslog\", mac=~\"$phone\"} [$__range])))",
|
||||||
|
"queryType": "instant"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"options": {
|
||||||
|
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||||
|
"colorMode": "none",
|
||||||
|
"graphMode": "none",
|
||||||
|
"textMode": "auto"
|
||||||
|
},
|
||||||
|
"fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "fixed", "fixedColor": "text" } }, "overrides": [] }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "stat",
|
||||||
|
"title": "Log lines",
|
||||||
|
"description": "Total lines from MAC-identified phones in range.",
|
||||||
|
"gridPos": { "h": 4, "w": 4, "x": 4, "y": 5 },
|
||||||
|
"id": 2,
|
||||||
|
"datasource": { "type": "loki", "uid": "${datasource}" },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"refId": "A",
|
||||||
|
"datasource": { "type": "loki", "uid": "${datasource}" },
|
||||||
|
"expr": "sum(count_over_time({job=\"syslog\", mac=~\"$phone\"} [$__range]))",
|
||||||
|
"queryType": "instant"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"options": {
|
||||||
|
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||||
|
"colorMode": "none",
|
||||||
|
"graphMode": "none",
|
||||||
|
"textMode": "auto"
|
||||||
|
},
|
||||||
|
"fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "fixed", "fixedColor": "text" } }, "overrides": [] }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "stat",
|
||||||
|
"title": "Active LDAP failures (15m)",
|
||||||
|
"description": "Distinct phones that could not resolve or contact the LDAP directory in the last 15 minutes. Verbose evidence identifies short-name DNS resolution of SERVERPDC as the cause. This can break directory/caller-name lookup, but not SIP calling itself.",
|
||||||
|
"gridPos": { "h": 4, "w": 4, "x": 8, "y": 5 },
|
||||||
|
"id": 3,
|
||||||
|
"datasource": { "type": "loki", "uid": "${datasource}" },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"refId": "A",
|
||||||
|
"datasource": { "type": "loki", "uid": "${datasource}" },
|
||||||
|
"expr": "count(count by (mac) (count_over_time({job=\"syslog\", mac=~\"$phone\"} |~ \"Domain name resolution failed|ldap_sasl_bind_s|Can't contact LDAP server\" [15m])))",
|
||||||
|
"queryType": "instant"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"options": {
|
||||||
|
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||||
|
"colorMode": "value",
|
||||||
|
"graphMode": "none",
|
||||||
|
"textMode": "auto"
|
||||||
|
},
|
||||||
|
"fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 1 }] } }, "overrides": [] }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "stat",
|
||||||
|
"title": "Recent provisioning failures (1h)",
|
||||||
|
"description": "Distinct phones with TFTP/provisioning download failures in the last hour. A short burst during reboot or fleet rollout is expected; a value that remains non-zero deserves investigation.",
|
||||||
|
"gridPos": { "h": 4, "w": 4, "x": 12, "y": 5 },
|
||||||
|
"id": 4,
|
||||||
|
"datasource": { "type": "loki", "uid": "${datasource}" },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"refId": "A",
|
||||||
|
"datasource": { "type": "loki", "uid": "${datasource}" },
|
||||||
|
"expr": "count(count by (mac) (count_over_time({job=\"syslog\", mac=~\"$phone\"} |~ \"tftp to file failed|no config Provisioning Server|DURL<3[+]error\" [1h])))",
|
||||||
|
"queryType": "instant"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"options": {
|
||||||
|
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||||
|
"colorMode": "value",
|
||||||
|
"graphMode": "none",
|
||||||
|
"textMode": "auto"
|
||||||
|
},
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "short",
|
||||||
|
"color": { "mode": "thresholds" },
|
||||||
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 1 }] }
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "stat",
|
||||||
|
"title": "Web password rejects",
|
||||||
|
"description": "Phones logging CUIT 'check passwd err' in the selected range. This is a rejected login to the handset web/UI, not SIP authentication; occasional entries are usually operator mistakes or background probing.",
|
||||||
|
"gridPos": { "h": 4, "w": 4, "x": 16, "y": 5 },
|
||||||
|
"id": 5,
|
||||||
|
"datasource": { "type": "loki", "uid": "${datasource}" },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"refId": "A",
|
||||||
|
"datasource": { "type": "loki", "uid": "${datasource}" },
|
||||||
|
"expr": "count(count by (mac) (count_over_time({job=\"syslog\", mac=~\"$phone\"} |~ \"check passwd err\" [$__range])))",
|
||||||
|
"queryType": "instant"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"options": {
|
||||||
|
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||||
|
"colorMode": "none",
|
||||||
|
"graphMode": "none",
|
||||||
|
"textMode": "auto"
|
||||||
|
},
|
||||||
|
"fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "fixed", "fixedColor": "text" } }, "overrides": [] }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "stat",
|
||||||
|
"title": "Internal DB faults (15m)",
|
||||||
|
"description": "Phones reporting a malformed internal database or related GUI virtual-machine failures in the last 15 minutes. Unlike ordinary firmware chatter, a persistent value can affect local phone applications and should be checked.",
|
||||||
|
"gridPos": { "h": 4, "w": 4, "x": 20, "y": 5 },
|
||||||
|
"id": 6,
|
||||||
|
"datasource": { "type": "loki", "uid": "${datasource}" },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"refId": "A",
|
||||||
|
"datasource": { "type": "loki", "uid": "${datasource}" },
|
||||||
|
"expr": "count(count by (mac) (count_over_time({job=\"syslog\", mac=~\"$phone\"} |~ \"database disk image is malformed|Null Virtual Machine pointer|Invalid scalar query\" [15m])))",
|
||||||
|
"queryType": "instant"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"options": {
|
||||||
|
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||||
|
"colorMode": "value",
|
||||||
|
"graphMode": "none",
|
||||||
|
"textMode": "auto"
|
||||||
|
},
|
||||||
|
"fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 1 }] } }, "overrides": [] }
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
"type": "bargauge",
|
||||||
|
"title": "Phones by operational-signal events",
|
||||||
|
"description": "Top phones for the narrow signals this dashboard can interpret: LDAP DNS/connectivity, internal database faults, provisioning/download failures, and SIP DNS timeouts. Keyed on MAC so DHCP changes do not split identity.",
|
||||||
|
"gridPos": { "h": 9, "w": 12, "x": 0, "y": 9 },
|
||||||
|
"id": 7,
|
||||||
|
"datasource": { "type": "loki", "uid": "${datasource}" },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"refId": "A",
|
||||||
|
"datasource": { "type": "loki", "uid": "${datasource}" },
|
||||||
|
"expr": "topk(10, sum by (mac) (count_over_time({job=\"syslog\", mac=~\"$phone\"} |~ \"Domain name resolution failed|ldap_sasl_bind_s|Can't contact LDAP server|database disk image is malformed|Null Virtual Machine pointer|Invalid scalar query|tftp to file failed|no config Provisioning Server|DURL<3[+]error|Timeout while contacting DNS servers\" [$__range])))",
|
||||||
|
"queryType": "instant",
|
||||||
|
"legendFormat": "{{mac}}"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"options": {
|
||||||
|
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||||
|
"displayMode": "gradient",
|
||||||
|
"orientation": "horizontal",
|
||||||
|
"showUnfilled": true
|
||||||
|
},
|
||||||
|
"fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "continuous-GrYlRd" } }, "overrides": [] }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "bargauge",
|
||||||
|
"title": "Loudest phones by log volume",
|
||||||
|
"description": "Identifies handsets left at log level 6 (Informational). Yealink's default is level 3 (Error); a phone near the top here is verbose by configuration, not necessarily unhealthy — but it dominates disk and query cost.",
|
||||||
|
"gridPos": { "h": 9, "w": 12, "x": 12, "y": 9 },
|
||||||
|
"id": 8,
|
||||||
|
"datasource": { "type": "loki", "uid": "${datasource}" },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"refId": "A",
|
||||||
|
"datasource": { "type": "loki", "uid": "${datasource}" },
|
||||||
|
"expr": "topk(10, sum by (mac) (count_over_time({job=\"syslog\", mac=~\"$phone\"} [$__range])))",
|
||||||
|
"queryType": "instant",
|
||||||
|
"legendFormat": "{{mac}}"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"options": {
|
||||||
|
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||||
|
"displayMode": "gradient",
|
||||||
|
"orientation": "horizontal",
|
||||||
|
"showUnfilled": true
|
||||||
|
},
|
||||||
|
"fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "continuous-BlPu" } }, "overrides": [] }
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
"type": "timeseries",
|
||||||
|
"title": "Operational signals over time",
|
||||||
|
"description": "Narrow, interpretable signals separated from firmware error-severity noise. LDAP short-name DNS failure affects directory lookup; internal DB affects local apps; provisioning should settle after boot; SIP DNS timeouts can prevent registration.",
|
||||||
|
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 18 },
|
||||||
|
"id": 9,
|
||||||
|
"datasource": { "type": "loki", "uid": "${datasource}" },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"refId": "A",
|
||||||
|
"datasource": { "type": "loki", "uid": "${datasource}" },
|
||||||
|
"expr": "sum(count_over_time({job=\"syslog\", mac=~\"$phone\"} |~ \"Domain name resolution failed|ldap_sasl_bind_s|Can't contact LDAP server\" [$__interval]))",
|
||||||
|
"queryType": "range",
|
||||||
|
"legendFormat": "LDAP directory"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"refId": "B",
|
||||||
|
"datasource": { "type": "loki", "uid": "${datasource}" },
|
||||||
|
"expr": "sum(count_over_time({job=\"syslog\", mac=~\"$phone\"} |~ \"database disk image is malformed|Null Virtual Machine pointer|Invalid scalar query\" [$__interval]))",
|
||||||
|
"queryType": "range",
|
||||||
|
"legendFormat": "Internal DB / GUI"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"refId": "C",
|
||||||
|
"datasource": { "type": "loki", "uid": "${datasource}" },
|
||||||
|
"expr": "sum(count_over_time({job=\"syslog\", mac=~\"$phone\"} |~ \"tftp to file failed|no config Provisioning Server|DURL<3[+]error\" [$__interval]))",
|
||||||
|
"queryType": "range",
|
||||||
|
"legendFormat": "Provisioning / download"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"refId": "D",
|
||||||
|
"datasource": { "type": "loki", "uid": "${datasource}" },
|
||||||
|
"expr": "sum(count_over_time({job=\"syslog\", mac=~\"$phone\"} |~ \"Timeout while contacting DNS servers\" [$__interval]))",
|
||||||
|
"queryType": "range",
|
||||||
|
"legendFormat": "SIP DNS timeout"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"options": {
|
||||||
|
"legend": { "displayMode": "table", "placement": "right", "showLegend": true, "calcs": ["sum"] },
|
||||||
|
"tooltip": { "mode": "multi", "sort": "desc" }
|
||||||
|
},
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "short",
|
||||||
|
"custom": { "drawStyle": "bars", "lineWidth": 0, "fillOpacity": 80, "stacking": { "mode": "normal", "group": "A" }, "showPoints": "never" }
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "timeseries",
|
||||||
|
"title": "Diagnostic severity volume (not health)",
|
||||||
|
"description": "Shows what the firmware labeled by severity. Do not interpret the red area as failed phones: observed error-level traffic is dominated by benign internal diagnostics. Emergency is excluded because Yealink uses it for a boot-time log-level table.",
|
||||||
|
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 18 },
|
||||||
|
"id": 10,
|
||||||
|
"datasource": { "type": "loki", "uid": "${datasource}" },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"refId": "A",
|
||||||
|
"datasource": { "type": "loki", "uid": "${datasource}" },
|
||||||
|
"expr": "sum by (severity) (count_over_time({job=\"syslog\", mac=~\"$phone\", severity!=\"emergency\"} [$__interval]))",
|
||||||
|
"queryType": "range",
|
||||||
|
"legendFormat": "{{severity}}"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"options": {
|
||||||
|
"legend": { "displayMode": "list", "placement": "bottom", "showLegend": true },
|
||||||
|
"tooltip": { "mode": "multi", "sort": "desc" }
|
||||||
|
},
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "short",
|
||||||
|
"custom": { "drawStyle": "bars", "lineWidth": 0, "fillOpacity": 70, "stacking": { "mode": "normal", "group": "A" }, "showPoints": "never" }
|
||||||
|
},
|
||||||
|
"overrides": [
|
||||||
|
{ "matcher": { "id": "byName", "options": "error" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } }] },
|
||||||
|
{ "matcher": { "id": "byName", "options": "warning" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "orange" } }] },
|
||||||
|
{ "matcher": { "id": "byName", "options": "notice" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "blue" } }] },
|
||||||
|
{ "matcher": { "id": "byName", "options": "informational" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "green" } }] }
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
"type": "logs",
|
||||||
|
"title": "Operational signals — investigate these first",
|
||||||
|
"description": "LDAP DNS/connectivity, internal database faults, provisioning/download failures, and SIP DNS timeouts. LDAP affects directory features but not calls; verbose phones identify SERVERPDC short-name resolution as the current LDAP cause. Labels show stable MAC and current source IP.",
|
||||||
|
"gridPos": { "h": 10, "w": 24, "x": 0, "y": 26 },
|
||||||
|
"id": 11,
|
||||||
|
"datasource": { "type": "loki", "uid": "${datasource}" },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"refId": "A",
|
||||||
|
"datasource": { "type": "loki", "uid": "${datasource}" },
|
||||||
|
"expr": "{job=\"syslog\", mac=~\"$phone\"} |~ \"Domain name resolution failed|ldap_sasl_bind_s|Can't contact LDAP server|database disk image is malformed|Null Virtual Machine pointer|Invalid scalar query|tftp to file failed|no config Provisioning Server|DURL<3[+]error|Timeout while contacting DNS servers\"",
|
||||||
|
"queryType": "range"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"options": {
|
||||||
|
"showTime": true,
|
||||||
|
"showLabels": true,
|
||||||
|
"showCommonLabels": false,
|
||||||
|
"wrapLogMessage": true,
|
||||||
|
"enableLogDetails": true,
|
||||||
|
"dedupStrategy": "none",
|
||||||
|
"sortOrder": "Descending"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
"type": "logs",
|
||||||
|
"title": "Raw firmware error severity (diagnostic only)",
|
||||||
|
"description": "For per-phone troubleshooting after the operational panels. These lines are not all actionable: Yealink marks substantial internal/boot chatter as error. Only the very high-volume RTP capture signature is removed to keep the panel usable.",
|
||||||
|
"gridPos": { "h": 10, "w": 24, "x": 0, "y": 36 },
|
||||||
|
"id": 12,
|
||||||
|
"datasource": { "type": "loki", "uid": "${datasource}" },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"refId": "A",
|
||||||
|
"datasource": { "type": "loki", "uid": "${datasource}" },
|
||||||
|
"expr": "{job=\"syslog\", mac=~\"$phone\", severity=\"error\"} !~ \"rtpcap get len not enough\"",
|
||||||
|
"queryType": "range"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"options": {
|
||||||
|
"showTime": true,
|
||||||
|
"showLabels": true,
|
||||||
|
"showCommonLabels": false,
|
||||||
|
"wrapLogMessage": true,
|
||||||
|
"enableLogDetails": true,
|
||||||
|
"dedupStrategy": "exact",
|
||||||
|
"sortOrder": "Descending"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
"type": "timeseries",
|
||||||
|
"title": "SIP registration refresh (level-6 phones only)",
|
||||||
|
"description": "REG 'Register: update server' fires at info level, so ONLY handsets left at log level 6 appear here. A gap for such a phone means it lost registration to the PBX (192.168.0.8); an absent phone means it is at level 3, NOT that it is unregistered.",
|
||||||
|
"gridPos": { "h": 7, "w": 12, "x": 0, "y": 46 },
|
||||||
|
"id": 13,
|
||||||
|
"datasource": { "type": "loki", "uid": "${datasource}" },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"refId": "A",
|
||||||
|
"datasource": { "type": "loki", "uid": "${datasource}" },
|
||||||
|
"expr": "sum by (mac) (count_over_time({job=\"syslog\", mac=~\"$phone\"} |~ \"Register: update server\" [$__interval]))",
|
||||||
|
"queryType": "range",
|
||||||
|
"legendFormat": "{{mac}}"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"options": {
|
||||||
|
"legend": { "displayMode": "list", "placement": "bottom", "showLegend": true },
|
||||||
|
"tooltip": { "mode": "multi" }
|
||||||
|
},
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "short",
|
||||||
|
"custom": { "drawStyle": "points", "pointSize": 5, "lineWidth": 1, "fillOpacity": 0, "showPoints": "always" }
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "logs",
|
||||||
|
"title": "DHCP lease events (level-6 phones only)",
|
||||||
|
"description": "Lease acquisition/renewal as the phone sees it — notice level, so the same caveat as the registration panel. Useful when a handset is 'connected but dead'. Observed lease time is 1200 s, which is exactly why source_ip is not a safe identity.",
|
||||||
|
"gridPos": { "h": 7, "w": 12, "x": 12, "y": 46 },
|
||||||
|
"id": 14,
|
||||||
|
"datasource": { "type": "loki", "uid": "${datasource}" },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"refId": "A",
|
||||||
|
"datasource": { "type": "loki", "uid": "${datasource}" },
|
||||||
|
"expr": "{job=\"syslog\", mac=~\"$phone\"} |~ \"lease of .* obtained|lease renew\"",
|
||||||
|
"queryType": "range"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"options": {
|
||||||
|
"showTime": true,
|
||||||
|
"showLabels": true,
|
||||||
|
"showCommonLabels": false,
|
||||||
|
"wrapLogMessage": true,
|
||||||
|
"enableLogDetails": true,
|
||||||
|
"dedupStrategy": "none",
|
||||||
|
"sortOrder": "Descending"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"refresh": "5m",
|
||||||
|
"schemaVersion": 39,
|
||||||
|
"tags": ["yealink", "phones", "syslog", "02-tekhnohim"],
|
||||||
|
"templating": {
|
||||||
|
"list": [
|
||||||
|
{
|
||||||
|
"name": "datasource",
|
||||||
|
"label": "Datasource",
|
||||||
|
"type": "datasource",
|
||||||
|
"query": "loki",
|
||||||
|
"current": {},
|
||||||
|
"hide": 0,
|
||||||
|
"refresh": 1,
|
||||||
|
"regex": "",
|
||||||
|
"skipUrlSync": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "phone",
|
||||||
|
"label": "Phone (MAC)",
|
||||||
|
"description": "Device identity. allValue is '.+' (not '.*') so that 'All' matches only streams that HAVE a mac label — which excludes switches and other non-Yealink senders sharing this Loki.",
|
||||||
|
"type": "query",
|
||||||
|
"datasource": { "type": "loki", "uid": "${datasource}" },
|
||||||
|
"definition": "label_values({job=\"syslog\"}, mac)",
|
||||||
|
"query": {
|
||||||
|
"label": "mac",
|
||||||
|
"refId": "LokiVariableQueryEditor-VariableQuery",
|
||||||
|
"stream": "{job=\"syslog\"}",
|
||||||
|
"type": 1
|
||||||
|
},
|
||||||
|
"current": { "selected": true, "text": ["All"], "value": ["$__all"] },
|
||||||
|
"includeAll": true,
|
||||||
|
"multi": true,
|
||||||
|
"allValue": ".+",
|
||||||
|
"hide": 0,
|
||||||
|
"refresh": 2,
|
||||||
|
"sort": 1,
|
||||||
|
"skipUrlSync": false
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"time": { "from": "now-24h", "to": "now" },
|
||||||
|
"timepicker": {},
|
||||||
|
"timezone": "browser",
|
||||||
|
"title": "Yealink IP phones — syslog health",
|
||||||
|
"uid": "yealink-phones",
|
||||||
|
"version": 4,
|
||||||
|
"weekStart": ""
|
||||||
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,17 @@
|
|||||||
|
apiVersion: 1
|
||||||
|
|
||||||
|
# Loads every dashboard JSON from /var/lib/grafana/dashboards (bind-mounted from
|
||||||
|
# this repo's logging/grafana-dashboards/). Dashboards are file-provisioned, so
|
||||||
|
# they are recreated on every container start and edits made in the UI are not
|
||||||
|
# persisted — change the JSON in git and redeploy instead.
|
||||||
|
providers:
|
||||||
|
- name: 'logging-dashboards'
|
||||||
|
orgId: 1
|
||||||
|
folder: 'Network'
|
||||||
|
type: file
|
||||||
|
disableDeletion: false
|
||||||
|
updateIntervalSeconds: 30
|
||||||
|
allowUiUpdates: false
|
||||||
|
options:
|
||||||
|
path: /var/lib/grafana/dashboards
|
||||||
|
foldersFromFilesStructure: false
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
apiVersion: 1
|
||||||
|
|
||||||
|
datasources:
|
||||||
|
- name: Loki
|
||||||
|
type: loki
|
||||||
|
access: proxy
|
||||||
|
url: http://loki:3100
|
||||||
|
isDefault: true
|
||||||
|
editable: false
|
||||||
@@ -0,0 +1,66 @@
|
|||||||
|
auth_enabled: false
|
||||||
|
|
||||||
|
server:
|
||||||
|
http_listen_address: 0.0.0.0
|
||||||
|
http_listen_port: 3100
|
||||||
|
grpc_listen_address: 0.0.0.0
|
||||||
|
grpc_listen_port: 9096
|
||||||
|
log_level: info
|
||||||
|
|
||||||
|
common:
|
||||||
|
instance_addr: 127.0.0.1
|
||||||
|
path_prefix: /loki
|
||||||
|
storage:
|
||||||
|
filesystem:
|
||||||
|
chunks_directory: /loki/chunks
|
||||||
|
rules_directory: /loki/rules
|
||||||
|
replication_factor: 1
|
||||||
|
ring:
|
||||||
|
kvstore:
|
||||||
|
store: inmemory
|
||||||
|
|
||||||
|
schema_config:
|
||||||
|
configs:
|
||||||
|
- from: 2024-01-01
|
||||||
|
store: tsdb
|
||||||
|
object_store: filesystem
|
||||||
|
schema: v13
|
||||||
|
index:
|
||||||
|
prefix: index_
|
||||||
|
period: 24h
|
||||||
|
|
||||||
|
limits_config:
|
||||||
|
reject_old_samples: true
|
||||||
|
reject_old_samples_max_age: 168h
|
||||||
|
retention_period: 8760h # 365 days — required minimum; monitor disk usage as fleet onboarding grows (host has ~40GB free)
|
||||||
|
max_query_series: 5000
|
||||||
|
|
||||||
|
# --- Dashboard concurrency ---
|
||||||
|
# Default 30m split a 24h dashboard panel into 48 sub-queries. With 14 panels
|
||||||
|
# refreshing together that is ~672 sub-queries against a scheduler queue of
|
||||||
|
# 100 -> HTTP 429 "too many outstanding requests", and panels blank out at
|
||||||
|
# random on each refresh (the "data appears then disappears" symptom).
|
||||||
|
# Measured 2026-07-15: 14 identical queries fired sequentially = 14/14 OK;
|
||||||
|
# the same 14 fired concurrently = 5/14 OK, 9 rejected.
|
||||||
|
# This fleet produces ~35k lines/day, so splitting buys nothing here.
|
||||||
|
# 24h also matches the schema_config index period above.
|
||||||
|
split_queries_by_interval: 24h
|
||||||
|
# Default 32 is far too optimistic for this 2-core host.
|
||||||
|
max_query_parallelism: 8
|
||||||
|
|
||||||
|
query_scheduler:
|
||||||
|
# Default 100. Headroom so one dashboard refresh cannot overflow the queue.
|
||||||
|
max_outstanding_requests_per_tenant: 2048
|
||||||
|
|
||||||
|
compactor:
|
||||||
|
working_directory: /loki/compactor
|
||||||
|
compaction_interval: 10m
|
||||||
|
retention_enabled: true
|
||||||
|
retention_delete_delay: 2h
|
||||||
|
delete_request_store: filesystem
|
||||||
|
|
||||||
|
ruler:
|
||||||
|
storage:
|
||||||
|
type: local
|
||||||
|
local:
|
||||||
|
directory: /loki/rules
|
||||||
@@ -0,0 +1,50 @@
|
|||||||
|
# HTTP proxy for the tekhnohim Tailscale exit-node egress.
|
||||||
|
#
|
||||||
|
# This 3proxy instance serves an HTTP proxy (:3128) chained to the Tailscale
|
||||||
|
# userspace SOCKS5 (ts-proxy:1055) so clients exit the internet as the Hetzner
|
||||||
|
# exit-node IP (89.167.72.79). TCP only (no QUIC/HTTP3).
|
||||||
|
#
|
||||||
|
# The generic SOCKS5 (:1080) was RETIRED 2026-07-24: raw SOCKS5 is blocked by
|
||||||
|
# TSPU/DPI for the external Telegram use case. Telegram is now served by the
|
||||||
|
# `ts-lan-mtproto` container (Fake-TLS MTProto proxy) instead, which reuses the
|
||||||
|
# :1080 external forward. See the profile / mtproto diagnostic.
|
||||||
|
#
|
||||||
|
# Access model (LAN free, outside password):
|
||||||
|
# * LAN 192.168.0.0/22 -> authorised by source IP, NO password.
|
||||||
|
# * everyone else -> must authenticate (external clients arriving via
|
||||||
|
# the router port-forward keep their public source
|
||||||
|
# IP, so they fall through to the password rule).
|
||||||
|
# The username/password lives in a host-only file (0600, NOT in git), mounted
|
||||||
|
# separately and pulled in with the include below - same pattern as ts.env.
|
||||||
|
|
||||||
|
nserver 127.0.0.11
|
||||||
|
nscache 65536
|
||||||
|
timeouts 1 5 30 60 180 1800 15 60
|
||||||
|
log
|
||||||
|
|
||||||
|
# Account(s) for external, password-authenticated access (host-only secret):
|
||||||
|
include /etc/3proxy/3proxy.secret
|
||||||
|
|
||||||
|
# iponly authorises the LAN by address; strong forces login for the rest.
|
||||||
|
auth iponly strong
|
||||||
|
|
||||||
|
# Safety guard: never treat the router's own address as a trusted LAN client.
|
||||||
|
# If the port-forward ever SNATs external traffic to the gateway IP, this rule
|
||||||
|
# makes the proxy fail CLOSED (external denied) instead of becoming an open
|
||||||
|
# proxy. Verify after opening the port: an unauthenticated external client
|
||||||
|
# must be refused.
|
||||||
|
deny * 192.168.0.1
|
||||||
|
|
||||||
|
# LAN clients: allowed by source IP, no password. Egress via the Tailscale
|
||||||
|
# userspace SOCKS5 (a `parent` binds to the `allow` line directly above it).
|
||||||
|
allow * 192.168.0.0/22
|
||||||
|
parent 1000 socks5 ts-proxy 1055
|
||||||
|
|
||||||
|
# External clients: must present the configured account, same egress.
|
||||||
|
allow thproxy
|
||||||
|
parent 1000 socks5 ts-proxy 1055
|
||||||
|
|
||||||
|
# Anything else: denied.
|
||||||
|
deny *
|
||||||
|
|
||||||
|
proxy -p3128 -a -i0.0.0.0
|
||||||
@@ -0,0 +1,74 @@
|
|||||||
|
# Tailscale exit-node LAN proxy
|
||||||
|
|
||||||
|
LAN HTTP and SOCKS5 proxies that egress through the dedicated Hetzner Tailscale
|
||||||
|
exit node, so proxied clients appear on the internet as `89.167.72.79` without
|
||||||
|
routing the docker host itself through the tunnel.
|
||||||
|
|
||||||
|
- HTTP endpoint: `192.168.0.35:3128/tcp` (`tinyproxy`, HTTP/HTTPS forward proxy);
|
||||||
|
- SOCKS5 endpoint: `192.168.0.35:1080/tcp` (`3proxy`, LAN-only);
|
||||||
|
- both fronts accept LAN clients (`192.168.0.0/22`) and forward upstream over
|
||||||
|
SOCKS5 to the Tailscale container (`tinyproxy upstream socks5`, `3proxy parent
|
||||||
|
socks5`);
|
||||||
|
- `ts-proxy` runs Tailscale in **userspace** mode and sends outbound traffic
|
||||||
|
through the exit node `fedora-technohim` (`100.121.234.85`);
|
||||||
|
- userspace mode adds **no host route/firewall changes** — the docker host's own
|
||||||
|
services (NPM, KMS, IPsec) keep egressing via the normal gateway;
|
||||||
|
- both fronts carry **TCP only** — QUIC/HTTP3 (UDP) is not proxied, so clients
|
||||||
|
must disable browser QUIC for e.g. YouTube video, or run the Tailscale client
|
||||||
|
directly and use the exit node for a full (UDP-capable) tunnel;
|
||||||
|
- proxy access logs ship to the logging stack (Alloy → Loki) via the syslog
|
||||||
|
log-driver (`tag: tailscale-proxy` for HTTP, `tag: tailscale-socks5` for
|
||||||
|
SOCKS5); the Grafana dashboard `logging/grafana-dashboards/tailscale-proxy.json`
|
||||||
|
visualises the HTTP proxy.
|
||||||
|
|
||||||
|
The stack deliberately keeps credential-bearing / stateful files outside Git:
|
||||||
|
|
||||||
|
| Host path | Purpose | Required mode |
|
||||||
|
|---|---|---|
|
||||||
|
| `/mnt/containers/tailscale-proxy/ts.env` | `TS_AUTHKEY=<tailscale auth key>` | `0600`, owner `root:root` |
|
||||||
|
| `/mnt/containers/tailscale-proxy/state/` | Tailscale node state (persists identity across restarts) | dir, owner `root:root` |
|
||||||
|
| `/mnt/containers/tailscale-proxy/tinyproxy.conf` | tinyproxy config (also tracked in this directory as the source of truth) | `0644` |
|
||||||
|
| `/mnt/containers/tailscale-proxy/3proxy.cfg` | 3proxy SOCKS5 config (also tracked in this directory as the source of truth) | `0644` |
|
||||||
|
|
||||||
|
Before deploying: create `ts.env` on the host with a Tailscale auth key, and in
|
||||||
|
the Tailscale admin console approve the exit node and allow this node to use it.
|
||||||
|
Deploy this directory as a Portainer Git stack named `tailscale-proxy`, or run it
|
||||||
|
with Docker Compose using project name `tailscale-proxy`. The absolute
|
||||||
|
configuration files must already exist on the host before deployment.
|
||||||
|
|
||||||
|
Full build record and rationale (userspace vs TUN dead-ends, SOCKS5-vs-HTTP,
|
||||||
|
monitoring) is in ops-knowledge:
|
||||||
|
`diagnostics/2026-07-23-02-tekhnohim-docker-tailscale-exit-proxy-stack.md`.
|
||||||
|
|
||||||
|
## Telemt monitoring
|
||||||
|
|
||||||
|
Telemt exports Prometheus metrics on container port `9090`, published as
|
||||||
|
`http://192.168.0.35:9092/metrics` because host port `9090` is reserved by
|
||||||
|
Cockpit. The Telemt application whitelist permits only the client-02 Zabbix
|
||||||
|
server (`192.168.0.34/32`). In Zabbix, import Telemt's upstream
|
||||||
|
`tools/zbx_telemt_template.yaml`, link it to the Docker host, and set
|
||||||
|
`{$TELEMT_URL}` to the URL above.
|
||||||
|
|
||||||
|
Prometheus deliberately exposes only active-IP counts. The files in
|
||||||
|
`zabbix/` add the text key `telemt.active_ips.list` without publishing
|
||||||
|
Telemt's control API. A root timer enters only the container's network
|
||||||
|
namespace, reads `/v1/users`, discards links/secrets in memory, and writes a
|
||||||
|
sanitized username/IP list readable by the Zabbix agent:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
install -o root -g root -m 0755 zabbix/zabbix-telemt-active-ips \
|
||||||
|
/usr/local/libexec/zabbix-telemt-active-ips
|
||||||
|
install -o root -g root -m 0644 zabbix/zabbix-telemt-active-ips.service \
|
||||||
|
zabbix/zabbix-telemt-active-ips.timer /etc/systemd/system/
|
||||||
|
install -o root -g root -m 0644 zabbix/telemt-active-ips.conf \
|
||||||
|
/etc/zabbix/zabbix_agent2.d/telemt-active-ips.conf
|
||||||
|
systemctl daemon-reload
|
||||||
|
systemctl enable --now zabbix-telemt-active-ips.timer
|
||||||
|
systemctl start zabbix-telemt-active-ips.service
|
||||||
|
systemctl restart zabbix-agent2
|
||||||
|
```
|
||||||
|
|
||||||
|
Create a Zabbix agent item on `Fedora Kirochnaya` with key
|
||||||
|
`telemt.active_ips.list`, information type **Text**, a 30-second update
|
||||||
|
interval, and one-day history. IP addresses are operationally sensitive; do
|
||||||
|
not retain this item longer or expose it on unrestricted dashboards.
|
||||||
@@ -0,0 +1,66 @@
|
|||||||
|
services:
|
||||||
|
ts-proxy:
|
||||||
|
image: tailscale/tailscale:v1.98.9
|
||||||
|
container_name: ts-proxy
|
||||||
|
hostname: technohim-lan-proxy
|
||||||
|
restart: unless-stopped
|
||||||
|
env_file: ["/mnt/containers/tailscale-proxy/ts.env"]
|
||||||
|
environment:
|
||||||
|
TS_HOSTNAME: technohim-lan-proxy
|
||||||
|
TS_STATE_DIR: /var/lib/tailscale
|
||||||
|
TS_USERSPACE: "true"
|
||||||
|
TS_ACCEPT_DNS: "false"
|
||||||
|
TS_EXTRA_ARGS: "--exit-node=100.121.234.85 --reset"
|
||||||
|
TS_TAILSCALED_EXTRA_ARGS: "--socks5-server=0.0.0.0:1055 --outbound-http-proxy-listen=0.0.0.0:1056"
|
||||||
|
volumes: ["/mnt/containers/tailscale-proxy/state:/var/lib/tailscale:z"]
|
||||||
|
networks: [proxynet]
|
||||||
|
|
||||||
|
# 3proxy HTTP front (:3128), LAN passwordless and external password-
|
||||||
|
# authenticated, egressing via the Tailscale exit node. (The generic SOCKS5
|
||||||
|
# :1080 was retired 2026-07-24 — see below.)
|
||||||
|
proxy:
|
||||||
|
image: ghcr.io/3proxy/3proxy:latest
|
||||||
|
container_name: ts-lan-proxy
|
||||||
|
restart: unless-stopped
|
||||||
|
depends_on: [ts-proxy]
|
||||||
|
volumes:
|
||||||
|
- "/mnt/containers/tailscale-proxy/3proxy.cfg:/etc/3proxy/3proxy.cfg:ro,z"
|
||||||
|
- "/mnt/containers/tailscale-proxy/3proxy.secret:/etc/3proxy/3proxy.secret:ro,z"
|
||||||
|
ports:
|
||||||
|
- "192.168.0.35:3128:3128"
|
||||||
|
logging:
|
||||||
|
driver: syslog
|
||||||
|
options:
|
||||||
|
syslog-address: "udp://192.168.0.35:514"
|
||||||
|
syslog-format: rfc3164
|
||||||
|
tag: tailscale-proxy
|
||||||
|
networks: [proxynet]
|
||||||
|
|
||||||
|
# Telegram Fake-TLS MTProto proxy on :1080. Uses Telemt (the same proven
|
||||||
|
# implementation as the Hetzner endpoint) after alexbers/mtprotoproxy's
|
||||||
|
# Fake-TLS handshake was rejected by real clients (see the mtproto diagnostic).
|
||||||
|
# Only DC-bound traffic is chained out through the Tailscale userspace SOCKS5
|
||||||
|
# ([[upstreams]] socks5 ts-proxy:1055 in telemt.toml), so egress rides the
|
||||||
|
# Hetzner exit node; the client-facing hop is direct Fake-TLS.
|
||||||
|
# telemt.toml is host-only (0600, uid 65532/nonroot, NOT in git) - holds the secret.
|
||||||
|
telemt:
|
||||||
|
image: ghcr.io/telemt/telemt:3.4.25
|
||||||
|
container_name: ts-lan-telemt
|
||||||
|
restart: unless-stopped
|
||||||
|
depends_on: [ts-proxy]
|
||||||
|
environment:
|
||||||
|
NO_COLOR: "1" # cleaner `docker logs` (no ANSI escapes)
|
||||||
|
volumes:
|
||||||
|
- "/mnt/containers/tailscale-proxy/telemt/telemt.toml:/app/config.toml:ro,z"
|
||||||
|
ports:
|
||||||
|
- "192.168.0.35:1080:1080"
|
||||||
|
# Host :9090 belongs to Cockpit; expose Telemt metrics on :9092 instead.
|
||||||
|
- "192.168.0.35:9092:9090"
|
||||||
|
# Stays on json-file: telemt's tracing logs don't ship cleanly over the
|
||||||
|
# syslog driver, and `docker logs ts-lan-telemt` is worth keeping for this
|
||||||
|
# critical service. Prometheus metrics are scraped by Zabbix on :9092.
|
||||||
|
networks: [proxynet]
|
||||||
|
|
||||||
|
networks:
|
||||||
|
proxynet:
|
||||||
|
name: tailscale-proxynet
|
||||||
@@ -0,0 +1,44 @@
|
|||||||
|
# Telemt config for the tekhnohim Telegram Fake-TLS MTProto proxy (docker host).
|
||||||
|
#
|
||||||
|
# The REAL telemt.toml is host-only at
|
||||||
|
# /mnt/containers/tailscale-proxy/telemt/telemt.toml (mode 0600, owned by the
|
||||||
|
# image's nonroot uid 65532, NOT in git) because [access.users] holds the
|
||||||
|
# proxy secret. This is the secret-free template.
|
||||||
|
#
|
||||||
|
# Egress: [[upstreams]] socks5 -> ts-proxy:1055 (Tailscale userspace SOCKS5),
|
||||||
|
# so ONLY Telegram-DC traffic exits via the Hetzner exit node (89.167.72.79).
|
||||||
|
# The client-facing side is direct Fake-TLS on :1080.
|
||||||
|
|
||||||
|
# Direct-to-DC mode: with a SOCKS5 upstream, only DC-bound traffic is chained,
|
||||||
|
# so use_middle_proxy MUST be false (middle-proxy/ME servers are not routed
|
||||||
|
# through the upstream and fail to init behind the Tailscale SOCKS5).
|
||||||
|
[general]
|
||||||
|
use_middle_proxy = false
|
||||||
|
|
||||||
|
[general.modes]
|
||||||
|
classic = false
|
||||||
|
secure = false
|
||||||
|
tls = true # ee / FakeTLS
|
||||||
|
|
||||||
|
[server]
|
||||||
|
port = 1080
|
||||||
|
# Host TCP :9090 is Cockpit, so compose maps host 192.168.0.35:9092 to this
|
||||||
|
# container port. Explicit 0.0.0.0 is required because metrics_port alone binds
|
||||||
|
# Telemt 3.4.25 to container loopback. Only the client-02 Zabbix server may scrape.
|
||||||
|
metrics_port = 9090
|
||||||
|
metrics_listen = "0.0.0.0:9090"
|
||||||
|
metrics_whitelist = ["192.168.0.34/32"]
|
||||||
|
|
||||||
|
[censorship]
|
||||||
|
tls_domain = "www.google.com" # SNI presented in the Fake-TLS handshake; tunable
|
||||||
|
|
||||||
|
[access.users]
|
||||||
|
# username = 32-hex-char secret (16 bytes). Generate: openssl rand -hex 16
|
||||||
|
tekhnohim = "00000000000000000000000000000000"
|
||||||
|
|
||||||
|
# Chain Telegram-DC connections out through the Tailscale userspace SOCKS5.
|
||||||
|
[[upstreams]]
|
||||||
|
type = "socks5"
|
||||||
|
address = "ts-proxy:1055"
|
||||||
|
weight = 1
|
||||||
|
enabled = true
|
||||||
@@ -0,0 +1,2 @@
|
|||||||
|
# Sanitized by zabbix-telemt-active-ips.service; contains no proxy secrets.
|
||||||
|
UserParameter=telemt.active_ips.list,cat /var/lib/zabbix/telemt-active-ips.txt
|
||||||
+52
@@ -0,0 +1,52 @@
|
|||||||
|
#!/usr/bin/bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
container_name="ts-lan-telemt"
|
||||||
|
output_dir="/var/lib/zabbix"
|
||||||
|
json_target="${output_dir}/telemt-active-ips.json"
|
||||||
|
text_target="${output_dir}/telemt-active-ips.txt"
|
||||||
|
|
||||||
|
install -d -o root -g zabbix -m 0750 "${output_dir}"
|
||||||
|
|
||||||
|
container_pid="$(docker inspect --format '{{.State.Pid}}' "${container_name}")"
|
||||||
|
if [[ -z "${container_pid}" || "${container_pid}" == "0" ]]; then
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
json_tmp="$(mktemp "${output_dir}/.telemt-active-ips.json.XXXXXX")"
|
||||||
|
text_tmp="$(mktemp "${output_dir}/.telemt-active-ips.txt.XXXXXX")"
|
||||||
|
trap 'rm -f "${json_tmp}" "${text_tmp}"' EXIT
|
||||||
|
|
||||||
|
# The API response also contains proxy links and secrets. Filter it in-memory
|
||||||
|
# and persist only the username and current active IP address list.
|
||||||
|
nsenter -t "${container_pid}" -n \
|
||||||
|
curl --fail --silent --show-error --max-time 5 \
|
||||||
|
http://127.0.0.1:9091/v1/users |
|
||||||
|
jq --compact-output '{
|
||||||
|
users: [
|
||||||
|
.data[]
|
||||||
|
| {
|
||||||
|
username,
|
||||||
|
active_unique_ips: (.active_unique_ips_list // [])
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}' > "${json_tmp}"
|
||||||
|
|
||||||
|
{
|
||||||
|
active_count="$(jq '[.users[].active_unique_ips[]] | length' "${json_tmp}")"
|
||||||
|
printf 'Active IP addresses: %s\n' "${active_count}"
|
||||||
|
jq --raw-output '
|
||||||
|
[
|
||||||
|
.users[] as $user
|
||||||
|
| $user.active_unique_ips[]
|
||||||
|
| "\($user.username) | \(.)"
|
||||||
|
]
|
||||||
|
| if length == 0 then ["No active IP addresses"] else . end
|
||||||
|
| .[]
|
||||||
|
' "${json_tmp}"
|
||||||
|
} > "${text_tmp}"
|
||||||
|
|
||||||
|
chown root:zabbix "${json_tmp}" "${text_tmp}"
|
||||||
|
chmod 0640 "${json_tmp}" "${text_tmp}"
|
||||||
|
mv -f "${json_tmp}" "${json_target}"
|
||||||
|
mv -f "${text_tmp}" "${text_target}"
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=Collect sanitized Telemt active IP addresses for Zabbix
|
||||||
|
After=docker.service
|
||||||
|
Requires=docker.service
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=oneshot
|
||||||
|
ExecStart=/usr/local/libexec/zabbix-telemt-active-ips
|
||||||
|
User=root
|
||||||
|
Group=root
|
||||||
|
PrivateTmp=true
|
||||||
|
NoNewPrivileges=true
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=Refresh sanitized Telemt active IP addresses for Zabbix
|
||||||
|
|
||||||
|
[Timer]
|
||||||
|
OnBootSec=30s
|
||||||
|
OnUnitActiveSec=30s
|
||||||
|
AccuracySec=2s
|
||||||
|
Unit=zabbix-telemt-active-ips.service
|
||||||
|
Persistent=true
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=timers.target
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
# Telegram proxy through Xray Reality
|
||||||
|
|
||||||
|
This stack publishes TeleProxy only on the client LAN:
|
||||||
|
|
||||||
|
- client endpoint: `192.168.0.35:10443/tcp`;
|
||||||
|
- TeleProxy runs in direct mode;
|
||||||
|
- Telegram DC traffic goes to the local Xray SOCKS5 client;
|
||||||
|
- Xray sends the outbound leg through the dedicated Hetzner VLESS+Reality
|
||||||
|
server at `89.167.72.79:443`;
|
||||||
|
- Nginx Proxy Manager keeps exclusive ownership of host ports `80` and `443`.
|
||||||
|
|
||||||
|
The stack deliberately keeps credential-bearing files outside Git:
|
||||||
|
|
||||||
|
| Host path | Purpose | Required mode |
|
||||||
|
|---|---|---|
|
||||||
|
| `/mnt/containers/telegram-reality/xray-client.json` | Xray Reality client credentials and local SOCKS inbound | `0600`, owner `65532:65532` |
|
||||||
|
| `/mnt/containers/telegram-reality/teleproxy.env` | TeleProxy secret and label | `0600`, owner `root:root` |
|
||||||
|
| `/mnt/containers/telegram-reality/teleproxy-link.txt` | Generated LAN-only Telegram connection link | `0600`, owner `root:root` |
|
||||||
|
|
||||||
|
Deploy this directory as a Portainer Git stack named `telegram-reality`, or run
|
||||||
|
it with Docker Compose using project name `telegram-reality`. The absolute
|
||||||
|
configuration files must already exist on the host before deployment.
|
||||||
@@ -0,0 +1,47 @@
|
|||||||
|
services:
|
||||||
|
xray-client:
|
||||||
|
image: ghcr.io/xtls/xray-core@sha256:592ec4d11f656db95598d01e76dbcc6e002d67360b96a5436500a938230f52c7
|
||||||
|
container_name: telegram-reality-xray
|
||||||
|
command:
|
||||||
|
- run
|
||||||
|
- -config
|
||||||
|
- /etc/xray/config.json
|
||||||
|
restart: unless-stopped
|
||||||
|
volumes:
|
||||||
|
- /mnt/containers/telegram-reality/xray-client.json:/etc/xray/config.json:ro,Z
|
||||||
|
networks:
|
||||||
|
- telegram-reality-nw
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "xray", "run", "-test", "-config", "/etc/xray/config.json"]
|
||||||
|
interval: 30s
|
||||||
|
timeout: 10s
|
||||||
|
retries: 3
|
||||||
|
start_period: 10s
|
||||||
|
|
||||||
|
teleproxy:
|
||||||
|
image: ghcr.io/teleproxy/teleproxy@sha256:f1d4c169d1a94a0d4871f9596fb2ca2783dc3015bcabe168ae59bc74c7a75e64
|
||||||
|
container_name: telegram-reality-teleproxy
|
||||||
|
restart: unless-stopped
|
||||||
|
depends_on:
|
||||||
|
xray-client:
|
||||||
|
condition: service_healthy
|
||||||
|
env_file:
|
||||||
|
- /mnt/containers/telegram-reality/teleproxy.env
|
||||||
|
environment:
|
||||||
|
DIRECT_MODE: "true"
|
||||||
|
SOCKS5_PROXY: "socks5://xray-client:1080"
|
||||||
|
PORT: "443"
|
||||||
|
EXTERNAL_PORT: "10443"
|
||||||
|
STATS_PORT: "8888"
|
||||||
|
WORKERS: "1"
|
||||||
|
EE_DOMAIN: "www.google.com"
|
||||||
|
EXTERNAL_IP: "89.167.72.79"
|
||||||
|
MSS_CLAMP: "false"
|
||||||
|
ports:
|
||||||
|
- "192.168.0.35:10443:443/tcp"
|
||||||
|
networks:
|
||||||
|
- telegram-reality-nw
|
||||||
|
|
||||||
|
networks:
|
||||||
|
telegram-reality-nw:
|
||||||
|
name: telegram-reality-nw
|
||||||
Reference in New Issue
Block a user