diff --git a/logging/README.md b/logging/README.md new file mode 100644 index 0000000..649c70a --- /dev/null +++ b/logging/README.md @@ -0,0 +1,126 @@ +# Internal logging stack + +Portainer Git stack for `outline` (`10.0.0.6`). It replaces only the logging +part of the manual `mktxp-stack`: + +- `loki` stores syslog in the persistent host path + `/mnt/containers/logging/loki-data`; +- `alloy` replaces the `syslog-ng` -> `promtail` relay; +- `grafana` provides the existing MikroTik Loki dashboard. + +The retired stack's `mktxp`, Prometheus, Promtail, and syslog-ng services are +not part of this stack. + +## Portainer settings + +- Repository: `https://git.it-depot.ru/depot/docker-infrastructure` +- Branch: `master` +- Compose path: `logging/docker-compose.yaml` +- Required environment variable: `GRAFANA_ADMIN_PASSWORD` + +Do not deploy the new stack before completing the cutover below: the current +containers still own names `loki` and `grafana` and host ports 3000, 3100, 514, +and 601. + +## Host files + +The Git-backed compose file intentionally bind-mounts stable host paths. +Before deployment, the repository files must be copied to: + +```text +/mnt/containers/logging/ +├── alloy-config/config.alloy +├── alloy-data/ +├── grafana-config/ +│ ├── dashboards/mikrotik-loki-logs.json +│ └── provisioning/ +│ ├── dashboards/dashboards.yaml +│ └── datasources/loki.yaml +├── grafana-data/ +├── loki-config/config.yaml +└── loki-data/ +``` + +SELinux is enforcing. The compose mounts use `:Z`; do not remove those flags. +The persistent data directories should be owned by the container users: +Loki `10001:10001`, Grafana `472:0`, and Alloy `0:0`. + +## Cutover + +The live Loki history is inside the old container at `/tmp/loki`. A stopped, +consistent rescue copy from 2026-07-30 is pre-seeded in the new host path, but +it must be refreshed from the stopped live container immediately before the +old stack is removed: + +```bash +cd /mnt/containers/mktxp/mktxp-stack + +# Record a last pre-cutover query and stop all writers cleanly. +curl -fsS http://127.0.0.1:3100/ready +docker compose stop + +# The stopped Loki container still contains the newest WAL/chunks. +docker cp -a loki:/tmp/loki/. /mnt/containers/logging/loki-data/ +chown -R 10001:10001 /mnt/containers/logging/loki-data + +# Removes containers and the old network, but NOT the old named Grafana and +# Prometheus volumes. Do not add --volumes. +docker compose down +``` + +Now deploy the Portainer Git stack. Expected interruption is the time between +`docker compose stop` and the new stack becoming ready. + +## Verification + +```bash +docker ps --filter name=loki --filter name=alloy --filter name=grafana +curl -fsS http://127.0.0.1:3100/ready +curl -fsS http://127.0.0.1:3000/api/health +ss -lnup | grep ':514' +ss -lntp | grep -E ':(601|3000|3100)\b' +``` + +Confirm both history and new ingestion: + +```bash +START=$(date -u -d '24 hours ago' +%s%N) +curl -fsSG http://127.0.0.1:3100/loki/api/v1/query_range \ + --data-urlencode 'query={job="syslog"}' \ + --data-urlencode "start=$START" \ + --data-urlencode 'limit=1' + +logger --server 127.0.0.1 --udp --port 514 \ + --rfc3164 --tag depot-cutover-test 'alloy ingestion test' +``` + +In Grafana, the provisioned **Network / Mikrotik Loki Logs** dashboard should +list existing `routerboard` values and show both pre-cutover and post-cutover +entries. + +## Rollback + +Remove the new Portainer stack, then recreate the old stack with its Loki data +bound back into `/tmp/loki`: + +```bash +cd /mnt/containers/mktxp/mktxp-stack +cp docker-compose.yaml docker-compose.rollback.yaml +``` + +Add this second volume to the `loki` service in +`docker-compose.rollback.yaml`: + +```yaml + - /mnt/containers/logging/loki-data:/tmp/loki:Z +``` + +Then run: + +```bash +docker compose -f docker-compose.rollback.yaml up -d +curl -fsS http://127.0.0.1:3100/ready +``` + +Do not restore the old Loki without that bind mount: its original writable +layer is deleted when the old stack is taken down. diff --git a/logging/alloy-config.alloy b/logging/alloy-config.alloy new file mode 100644 index 0000000..b5817d2 --- /dev/null +++ b/logging/alloy-config.alloy @@ -0,0 +1,65 @@ +// Direct replacement for the old syslog-ng -> Promtail -> Loki path. +// +// Existing RouterOS senders target 185.108.4.158:514/udp and use classic +// BSD/RFC3164 framing. TCP/601 is retained for RFC5424-capable senders. + +loki.relabel "syslog" { + forward_to = [] + + // Preserve the label used by the existing history and MikroTik dashboard. + rule { + source_labels = ["__syslog_message_hostname"] + target_label = "routerboard" + } + + rule { + source_labels = ["__syslog_message_severity"] + target_label = "severity" + } + + rule { + source_labels = ["__syslog_message_facility"] + target_label = "facility" + } + + rule { + source_labels = ["__syslog_connection_ip_address"] + target_label = "source_ip" + } +} + +loki.source.syslog "network_devices_udp" { + listener { + address = "0.0.0.0:514" + protocol = "udp" + syslog_format = "rfc3164" + labels = { + job = "syslog", + transport = "udp", + } + } + + relabel_rules = loki.relabel.syslog.rules + forward_to = [loki.write.default.receiver] +} + +loki.source.syslog "network_devices_tcp" { + listener { + address = "0.0.0.0:601" + protocol = "tcp" + syslog_format = "rfc5424" + labels = { + job = "syslog", + transport = "tcp", + } + } + + relabel_rules = loki.relabel.syslog.rules + forward_to = [loki.write.default.receiver] +} + +loki.write "default" { + endpoint { + url = "http://loki:3100/loki/api/v1/push" + } +} diff --git a/logging/docker-compose.yaml b/logging/docker-compose.yaml new file mode 100644 index 0000000..681118e --- /dev/null +++ b/logging/docker-compose.yaml @@ -0,0 +1,63 @@ +services: + loki: + # This pin and the TSDB v13 on-disk format were validated on Fedora 44 in + # the 02-tekhnohim logging stack. Do not replace it with :latest during + # the data migration. + image: grafana/loki:3.7.2 + container_name: loki + command: -config.file=/etc/loki/config.yaml + restart: unless-stopped + volumes: + - /mnt/containers/logging/loki-config/config.yaml:/etc/loki/config.yaml:ro,Z + - /mnt/containers/logging/loki-data:/loki:Z + ports: + # Retained for the existing operator query path. + - "3100:3100/tcp" + networks: + - logging-nw + + alloy: + # Replaces the syslog-ng -> Promtail relay with one direct syslog receiver. + image: grafana/alloy:v1.16.1 + container_name: alloy + command: + - run + - --server.http.listen-addr=0.0.0.0:12345 + - --storage.path=/var/lib/alloy/data + - /etc/alloy/config.alloy + restart: unless-stopped + depends_on: + - loki + volumes: + - /mnt/containers/logging/alloy-config/config.alloy:/etc/alloy/config.alloy:ro,Z + - /mnt/containers/logging/alloy-data:/var/lib/alloy/data:Z + ports: + # Existing fleet target: 185.108.4.158:514/udp. + - "514:514/udp" + # Preserve the old stack's TCP syslog endpoint. + - "601:601/tcp" + networks: + - logging-nw + + grafana: + image: grafana/grafana:13.1.0 + container_name: grafana + restart: unless-stopped + depends_on: + - loki + environment: + GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD} + GF_USERS_ALLOW_SIGN_UP: "false" + volumes: + - /mnt/containers/logging/grafana-data:/var/lib/grafana:Z + - /mnt/containers/logging/grafana-config/provisioning/datasources:/etc/grafana/provisioning/datasources:ro,Z + - /mnt/containers/logging/grafana-config/provisioning/dashboards:/etc/grafana/provisioning/dashboards:ro,Z + - /mnt/containers/logging/grafana-config/dashboards:/var/lib/grafana/dashboards:ro,Z + ports: + # Retained for the current LAN/operator access path. + - "3000:3000/tcp" + networks: + - logging-nw + +networks: + logging-nw: diff --git a/logging/grafana-dashboards/mikrotik-loki-logs.json b/logging/grafana-dashboards/mikrotik-loki-logs.json new file mode 100644 index 0000000..51f5be2 --- /dev/null +++ b/logging/grafana-dashboards/mikrotik-loki-logs.json @@ -0,0 +1,545 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "Mikrotik Loki Logs", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 17139, + "graphTooltip": 0, + "id": 3, + "links": [ + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": false, + "tags": [ + "mikrotik", + "mktxp" + ], + "targetBlank": false, + "title": "MKTXP Exporter", + "tooltip": "", + "type": "dashboards", + "url": "" + }, + { + "asDropdown": true, + "icon": "external link", + "keepTime": false, + "tags": [ + "system" + ], + "title": "System Overview", + "type": "dashboards" + } + ], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-orange", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 9, + "x": 0, + "y": 0 + }, + "id": 4, + "interval": "$smooth", + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.3.2", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "count_over_time({job=\"syslog\", routerboard=\"$routerboard\"} [$__interval] |~\"(?i)$topics\" |~\"(?i)$searchable_pattern\" !~\"(?i)$exclude\")", + "legendFormat": "{{host}}", + "queryType": "range", + "refId": "A" + } + ], + "title": "Log Counts By Topic", + "transformations": [], + "type": "stat" + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 15, + "x": 9, + "y": 0 + }, + "id": 3, + "interval": "$smooth", + "options": { + "legend": { + "calcs": [ + "mean", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "none", + "sort": "none" + } + }, + "pluginVersion": "8.1.2", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "count_over_time({job=\"syslog\", routerboard=\"$routerboard\"} [$__interval] |regexp \"(?P\\\\w+)\\\\: \" |~\"(?i)$topics\" |~\"(?i)$searchable_pattern\" !~\"(?i)$exclude\")", + "legendFormat": "{{prefix}}", + "queryType": "range", + "refId": "A" + } + ], + "title": "Log Counts By Topic Over Time", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "description": "", + "gridPos": { + "h": 23, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 5, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": false, + "sortOrder": "Descending", + "wrapLogMessage": false + }, + "pluginVersion": "7.3.6", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "code", + "expr": "{job=\"syslog\", routerboard=\"$routerboard\"} |~\"(?i)$topics\" |~\"(?i)$searchable_pattern\" !~\"(?i)$exclude\"", + "legendFormat": "", + "queryType": "range", + "refId": "A" + } + ], + "title": "Logs Lines By Topic", + "type": "logs" + } + ], + "refresh": "5s", + "schemaVersion": 37, + "style": "dark", + "tags": [ + "mikrotik", + "loki" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "MKT-GT", + "value": "MKT-GT" + }, + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "definition": "", + "hide": 0, + "includeAll": false, + "label": "Routerboard", + "multi": false, + "name": "routerboard", + "options": [], + "query": { + "label": "routerboard", + "refId": "LokiVariableQueryEditor-VariableQuery", + "stream": "", + "type": 1 + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "hide": 0, + "includeAll": true, + "label": "Topics", + "multi": true, + "name": "topics", + "options": [ + { + "selected": true, + "text": "All", + "value": "$__all" + }, + { + "selected": false, + "text": "account", + "value": "account" + }, + { + "selected": false, + "text": "caps", + "value": "caps" + }, + { + "selected": false, + "text": "critical", + "value": "critical" + }, + { + "selected": false, + "text": "error", + "value": "error" + }, + { + "selected": false, + "text": "firewall", + "value": "firewall" + }, + { + "selected": false, + "text": "info", + "value": "info" + }, + { + "selected": false, + "text": "warning", + "value": "warning" + }, + { + "selected": false, + "text": "wireless", + "value": "wireless" + }, + { + "selected": false, + "text": "dns", + "value": "dns" + }, + { + "selected": false, + "text": "health", + "value": "health" + }, + { + "selected": false, + "text": "dhcp", + "value": "dhcp" + }, + { + "selected": false, + "text": "interface", + "value": "interface" + }, + { + "selected": false, + "text": "ipsec", + "value": "ipsec" + }, + { + "selected": false, + "text": "l2tp", + "value": "l2tp" + }, + { + "selected": false, + "text": "ppp", + "value": "ppp" + }, + { + "selected": false, + "text": "raw", + "value": "raw" + }, + { + "selected": false, + "text": "ssh", + "value": "ssh" + }, + { + "selected": false, + "text": "system", + "value": "system" + }, + { + "selected": false, + "text": "ups", + "value": "ups" + } + ], + "query": "account,caps,critical,error,firewall,info,warning,wireless,dns,health,dhcp,interface,ipsec,l2tp,ppp,raw,ssh,system,ups", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + }, + { + "current": { + "selected": true, + "text": "", + "value": "" + }, + "description": "Search (case insensitive)", + "hide": 0, + "label": "Search", + "name": "searchable_pattern", + "options": [ + { + "selected": true, + "text": "", + "value": "" + } + ], + "query": "", + "skipUrlSync": false, + "type": "textbox" + }, + { + "current": { + "selected": false, + "text": "None", + "value": "None" + }, + "description": "Exclude pattern (case-insensitive, None includes everything)", + "hide": 0, + "includeAll": false, + "label": "Exclude", + "multi": false, + "name": "exclude", + "options": [ + { + "selected": true, + "text": "None", + "value": "None" + } + ], + "query": "None,", + "queryValue": "None", + "skipUrlSync": false, + "type": "custom" + }, + { + "current": { + "selected": false, + "text": "5m", + "value": "5m" + }, + "hide": 0, + "includeAll": false, + "label": "Smooth", + "multi": false, + "name": "smooth", + "options": [ + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "2m", + "value": "2m" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "15m", + "value": "15m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "2h", + "value": "2h" + } + ], + "query": "30s,1m,2m,5m,10m,15m,30m,1h,2h", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-12h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Mikrotik Loki Logs", + "uid": "awslix6doiakp-mkt-loki", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/logging/grafana-provisioning/dashboards/dashboards.yaml b/logging/grafana-provisioning/dashboards/dashboards.yaml new file mode 100644 index 0000000..fda94c4 --- /dev/null +++ b/logging/grafana-provisioning/dashboards/dashboards.yaml @@ -0,0 +1,13 @@ +apiVersion: 1 + +providers: + - name: logging-dashboards + orgId: 1 + folder: Network + type: file + disableDeletion: false + updateIntervalSeconds: 30 + allowUiUpdates: false + options: + path: /var/lib/grafana/dashboards + foldersFromFilesStructure: false diff --git a/logging/grafana-provisioning/datasources/loki.yaml b/logging/grafana-provisioning/datasources/loki.yaml new file mode 100644 index 0000000..bacda09 --- /dev/null +++ b/logging/grafana-provisioning/datasources/loki.yaml @@ -0,0 +1,10 @@ +apiVersion: 1 + +datasources: + - name: Loki + uid: P8E80F9AEF21F6940 + type: loki + access: proxy + url: http://loki:3100 + isDefault: true + editable: false diff --git a/logging/loki-config.yaml b/logging/loki-config.yaml new file mode 100644 index 0000000..14b5119 --- /dev/null +++ b/logging/loki-config.yaml @@ -0,0 +1,56 @@ +auth_enabled: false + +server: + http_listen_address: 0.0.0.0 + http_listen_port: 3100 + grpc_listen_address: 0.0.0.0 + grpc_listen_port: 9096 + log_level: info + +common: + instance_addr: 127.0.0.1 + path_prefix: /loki + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +schema_config: + configs: + # Keep the old stack's schema start and TSDB v13 format so its existing + # /tmp/loki data can be mounted at /loki without conversion. + - from: 2020-05-15 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +limits_config: + reject_old_samples: true + reject_old_samples_max_age: 168h + retention_period: 8760h + max_query_series: 5000 + split_queries_by_interval: 24h + max_query_parallelism: 16 + +query_scheduler: + max_outstanding_requests_per_tenant: 2048 + +compactor: + working_directory: /loki/compactor + compaction_interval: 10m + retention_enabled: true + retention_delete_delay: 2h + delete_request_store: filesystem + +ruler: + storage: + type: local + local: + directory: /loki/rules