diff --git a/logging/loki-config.yaml b/logging/loki-config.yaml index 07196d7..341744f 100644 --- a/logging/loki-config.yaml +++ b/logging/loki-config.yaml @@ -35,6 +35,23 @@ limits_config: retention_period: 8760h # 365 days — required minimum; monitor disk usage as fleet onboarding grows (host has ~40GB free) max_query_series: 5000 + # --- Dashboard concurrency --- + # Default 30m split a 24h dashboard panel into 48 sub-queries. With 14 panels + # refreshing together that is ~672 sub-queries against a scheduler queue of + # 100 -> HTTP 429 "too many outstanding requests", and panels blank out at + # random on each refresh (the "data appears then disappears" symptom). + # Measured 2026-07-15: 14 identical queries fired sequentially = 14/14 OK; + # the same 14 fired concurrently = 5/14 OK, 9 rejected. + # This fleet produces ~35k lines/day, so splitting buys nothing here. + # 24h also matches the schema_config index period above. + split_queries_by_interval: 24h + # Default 32 is far too optimistic for this 2-core host. + max_query_parallelism: 8 + +query_scheduler: + # Default 100. Headroom so one dashboard refresh cannot overflow the queue. + max_outstanding_requests_per_tenant: 2048 + compactor: working_directory: /loki/compactor compaction_interval: 10m