DEPOT logging: add Telemt fleet monitoring via Zabbix

Install and persist Grafana Zabbix plugin 6.5.0, provision the internal Zabbix datasource, and add a fleet dashboard modeled on client 02 Telemt panels.

Add an active-agent Zabbix 7 template plus a 30-second sanitized collector for active IP count/list, connections, traffic, and uptime. Active checks avoid inbound polling of client-private Docker hosts.

The exact Zabbix host names and required ZABBIX_API_TOKEN Portainer variable are documented. Compose validation passed on 10.0.0.6; the live plugin registered successfully under Grafana 13.1.0.

logging/zabbix-agent/telemt.conf

@@ -0,0 +1,8 @@
+# Cached by zabbix-telemt-collector.service; contains no proxy links or secrets.
+UserParameter=telemt.active_ips.list,cat /var/lib/zabbix/telemt-active-ips.txt
+UserParameter=telemt.active_ips.count,/usr/local/libexec/zabbix-telemt-value active_ips_count
+UserParameter=telemt.connections.current,/usr/local/libexec/zabbix-telemt-value connections_current
+UserParameter=telemt.connections.total,/usr/local/libexec/zabbix-telemt-value connections_total
+UserParameter=telemt.bytes.received,/usr/local/libexec/zabbix-telemt-value bytes_received
+UserParameter=telemt.bytes.sent,/usr/local/libexec/zabbix-telemt-value bytes_sent
+UserParameter=telemt.uptime,/usr/local/libexec/zabbix-telemt-value uptime

logging/zabbix-agent/zabbix-telemt-collector

@@ -0,0 +1,65 @@
+#!/usr/bin/bash
+set -euo pipefail
+
+container_name="ts-lan-telemt"
+output_dir="/var/lib/zabbix"
+json_target="${output_dir}/telemt-active-ips.json"
+text_target="${output_dir}/telemt-active-ips.txt"
+metrics_target="${output_dir}/telemt-metrics.prom"
+
+install -d -o root -g zabbix -m 0750 "${output_dir}"
+
+container_pid="$(docker inspect --format '{{.State.Pid}}' "${container_name}")"
+if [[ -z "${container_pid}" || "${container_pid}" == "0" ]]; then
+  exit 1
+fi
+
+host_ip="$(ip -4 route get 1.1.1.1 | awk '{for (i=1; i<=NF; i++) if ($i == "src") {print $(i+1); exit}}')"
+if [[ -z "${host_ip}" ]]; then
+  exit 1
+fi
+
+json_tmp="$(mktemp "${output_dir}/.telemt-active-ips.json.XXXXXX")"
+text_tmp="$(mktemp "${output_dir}/.telemt-active-ips.txt.XXXXXX")"
+metrics_tmp="$(mktemp "${output_dir}/.telemt-metrics.prom.XXXXXX")"
+trap 'rm -f "${json_tmp}" "${text_tmp}" "${metrics_tmp}"' EXIT
+
+# The API response also contains proxy links and secrets. Filter it in-memory
+# and persist only the username and current active IP address list.
+nsenter -t "${container_pid}" -n \
+  curl --fail --silent --show-error --max-time 5 \
+  http://127.0.0.1:9091/v1/users |
+  jq --compact-output '{
+    users: [
+      .data[]
+      | {
+          username,
+          active_unique_ips: (.active_unique_ips_list // [])
+        }
+    ]
+  }' > "${json_tmp}"
+
+{
+  active_count="$(jq '[.users[].active_unique_ips[]] | unique | length' "${json_tmp}")"
+  printf 'Active IP addresses: %s\n' "${active_count}"
+  jq --raw-output '
+    [
+      .users[] as $user
+      | $user.active_unique_ips[]
+      | "\($user.username) | \(.)"
+    ]
+    | if length == 0 then ["No active IP addresses"] else . end
+    | .[]
+  ' "${json_tmp}"
+} > "${text_tmp}"
+
+# Cache Prometheus output locally. Zabbix active checks read the cache, so the
+# central server does not need a route to each private Docker host.
+curl --fail --silent --show-error --max-time 5 \
+  "http://${host_ip}:9092/metrics" > "${metrics_tmp}"
+
+chown root:zabbix "${json_tmp}" "${text_tmp}" "${metrics_tmp}"
+chmod 0640 "${json_tmp}" "${text_tmp}" "${metrics_tmp}"
+mv -f "${json_tmp}" "${json_target}"
+mv -f "${text_tmp}" "${text_target}"
+mv -f "${metrics_tmp}" "${metrics_target}"

logging/zabbix-agent/zabbix-telemt-collector.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=Collect sanitized Telemt data for Zabbix
+After=docker.service
+Requires=docker.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/libexec/zabbix-telemt-collector
+User=root
+Group=root
+PrivateTmp=true
+NoNewPrivileges=true

logging/zabbix-agent/zabbix-telemt-collector.timer

@@ -0,0 +1,12 @@
+[Unit]
+Description=Refresh sanitized Telemt data for Zabbix
+
+[Timer]
+OnBootSec=30s
+OnUnitActiveSec=30s
+AccuracySec=2s
+Unit=zabbix-telemt-collector.service
+Persistent=true
+
+[Install]
+WantedBy=timers.target

logging/zabbix-agent/zabbix-telemt-value

@@ -0,0 +1,30 @@
+#!/usr/bin/bash
+set -euo pipefail
+
+json_file="/var/lib/zabbix/telemt-active-ips.json"
+metrics_file="/var/lib/zabbix/telemt-metrics.prom"
+
+case "${1:-}" in
+  active_ips_count)
+    jq '[.users[].active_unique_ips[]] | unique | length' "${json_file}"
+    ;;
+  connections_current)
+    awk '$1 ~ /^telemt_user_connections_current\{/ {sum += $2} END {print sum + 0}' "${metrics_file}"
+    ;;
+  connections_total)
+    awk '$1 ~ /^telemt_user_connections_total\{/ {sum += $2} END {print sum + 0}' "${metrics_file}"
+    ;;
+  bytes_received)
+    awk '$1 ~ /^telemt_user_octets_from_client\{/ {sum += $2} END {printf "%.0f\n", sum + 0}' "${metrics_file}"
+    ;;
+  bytes_sent)
+    awk '$1 ~ /^telemt_user_octets_to_client\{/ {sum += $2} END {printf "%.0f\n", sum + 0}' "${metrics_file}"
+    ;;
+  uptime)
+    awk '$1 == "telemt_uptime_seconds" {printf "%.0f\n", $2; found=1} END {if (!found) exit 1}' "${metrics_file}"
+    ;;
+  *)
+    printf 'unsupported Telemt value: %s\n' "${1:-}" >&2
+    exit 2
+    ;;
+esac