logsite 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
logsite/__init__.py ADDED
@@ -0,0 +1,18 @@
1
+ """logsite: production log volume and log-quality checks, inline next to your log calls."""
2
+
3
+ from .index import index_project, index_source
4
+ from .lint import LintConfig, lint_source
5
+ from .matching import Matcher
6
+ from .models import Diagnostic, LogRecord, LogSite
7
+
8
+ __all__ = [
9
+ "Diagnostic",
10
+ "LintConfig",
11
+ "LogRecord",
12
+ "LogSite",
13
+ "Matcher",
14
+ "index_project",
15
+ "index_source",
16
+ "lint_source",
17
+ ]
18
+ __version__ = "0.1.0"
logsite/__main__.py ADDED
@@ -0,0 +1,3 @@
1
+ from .cli import main
2
+
3
+ raise SystemExit(main())
logsite/alerts.py ADDED
@@ -0,0 +1,382 @@
1
+ """Alert enrichment: when Grafana fires an alert, send the error summary with it.
2
+
3
+ ``logsite alert-server`` receives Grafana's webhook contact point, summarizes the errors in
4
+ the alert's window (grouped by cause, with the line of code that raised them), and posts it
5
+ to Slack or another webhook. ``GET /errors`` serves the same summary as JSON.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import json
11
+ import logging
12
+ import os
13
+ import re
14
+ import secrets
15
+ import urllib.request
16
+ from collections.abc import Mapping
17
+ from dataclasses import dataclass, field
18
+ from datetime import datetime, timedelta, timezone
19
+ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
20
+ from typing import Any
21
+ from urllib.parse import parse_qs, urlparse
22
+
23
+ from .models import parse_iso
24
+
25
+ log = logging.getLogger(__name__)
26
+
27
+ MAX_BODY = 5 * 1024 * 1024 # Grafana batches alerts; this is plenty
28
+
29
+
30
+ @dataclass
31
+ class AlertsConfig:
32
+ slack_webhook: str | None = None # Slack incoming webhook URL
33
+ webhook: str | None = None # any URL: gets the summary as JSON
34
+ token: str | None = None # required as "Authorization: Bearer <token>" or ?token=
35
+ lookback_minutes: float = 30.0 # summarize from this long before the alert started
36
+ top: int = 3 # error groups per alert
37
+ selector_labels: list[str] = field(default_factory=lambda: ["service_name"])
38
+
39
+ @classmethod
40
+ def from_dict(cls, data: Mapping) -> AlertsConfig:
41
+ known = set(cls.__dataclass_fields__) | {
42
+ f"{k}_env" for k in ("slack_webhook", "webhook", "token")
43
+ }
44
+ unknown = set(data) - known
45
+ if unknown:
46
+ raise ValueError(f"unknown [alerts] key(s) {sorted(unknown)}")
47
+ values = {k: v for k, v in data.items() if not k.endswith("_env")}
48
+ for key in ("slack_webhook", "webhook", "token"):
49
+ env = data.get(f"{key}_env")
50
+ if env and values.get(key) is None:
51
+ values[key] = os.environ.get(env)
52
+ cfg = cls(**values)
53
+ cfg.selector_labels = [str(x) for x in cfg.selector_labels]
54
+ return cfg
55
+
56
+
57
+ @dataclass
58
+ class Alert:
59
+ name: str
60
+ status: str # firing | resolved
61
+ labels: dict[str, str]
62
+ starts_at: datetime | None
63
+ url: str | None # the alert rule in Grafana
64
+ dashboard_url: str | None
65
+ summary: str | None # the rule's summary / description annotation
66
+ silence_url: str | None = None
67
+
68
+
69
+ def parse_grafana(payload: Mapping[str, Any]) -> list[Alert]:
70
+ """The alerts in a Grafana (unified alerting) or Alertmanager webhook payload."""
71
+ alerts = []
72
+ for a in payload.get("alerts") or []:
73
+ if not isinstance(a, dict):
74
+ continue
75
+ labels = {str(k): str(v) for k, v in (a.get("labels") or {}).items()}
76
+ annotations = a.get("annotations") or {}
77
+ started = a.get("startsAt")
78
+ try:
79
+ starts_at = parse_iso(started) if started and not started.startswith("0001") else None
80
+ except ValueError:
81
+ starts_at = None
82
+ alerts.append(
83
+ Alert(
84
+ name=labels.get("alertname", "alert"),
85
+ status=str(a.get("status", payload.get("status", "firing"))),
86
+ labels=labels,
87
+ starts_at=starts_at,
88
+ url=a.get("generatorURL") or None,
89
+ dashboard_url=a.get("panelURL") or a.get("dashboardURL") or None,
90
+ summary=annotations.get("summary") or annotations.get("description"),
91
+ silence_url=a.get("silenceURL") or None,
92
+ )
93
+ )
94
+ return alerts
95
+
96
+
97
+ _PACKAGE = re.compile(r"^(?:[a-z_$][\w$]*\.)+(?=[A-Z])") # "com.example.orders." in a class name
98
+ _LABEL_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
99
+
100
+
101
+ def narrow_selector(selector: str, labels: Mapping[str, str], names: list[str]) -> str:
102
+ """Add the alert's labels (those in ``names``) to a LogQL stream selector, so the
103
+ summary covers the service that alerted: ``{env="prod"}`` + service_name=checkout ->
104
+ ``{env="prod", service_name="checkout"}``. Labels already in the selector are kept."""
105
+ body = selector.strip()
106
+ if not (body.startswith("{") and body.endswith("}")):
107
+ return selector
108
+ inner = body[1:-1].strip()
109
+ present = set(re.findall(r"([A-Za-z_][A-Za-z0-9_]*)\s*(?:=~|!=|!~|=)", inner))
110
+ extra = [
111
+ f"{n}={json.dumps(labels[n])}"
112
+ for n in names
113
+ if n in labels and n not in present and _LABEL_NAME.match(n)
114
+ ]
115
+ if not extra:
116
+ return selector
117
+ return "{" + ", ".join([inner, *extra] if inner else extra) + "}"
118
+
119
+
120
+ def window_for(alert: Alert, lookback_minutes: float, now: datetime) -> tuple[datetime, datetime]:
121
+ start = (alert.starts_at or now) - timedelta(minutes=lookback_minutes)
122
+ return min(start, now - timedelta(minutes=1)), now
123
+
124
+
125
+ # --- messages ---------------------------------------------------------------------------------
126
+
127
+
128
+ def _when(ts: datetime | None) -> str:
129
+ return ts.astimezone(timezone.utc).strftime("%H:%M UTC") if ts else "?"
130
+
131
+
132
+ def _link(text: str, url: str | None, slack: bool) -> str:
133
+ if not url:
134
+ return f"`{text}`" if slack else text
135
+ return f"<{url}|{text}>" if slack else f"{text} ({url})"
136
+
137
+
138
+ def render(
139
+ alert: Alert,
140
+ summary: dict,
141
+ *,
142
+ slack: bool,
143
+ code_line: Any = None,
144
+ code_url: Any = None,
145
+ trace_field: str = "trace_id",
146
+ ) -> str:
147
+ """The message for one alert: ``summary`` is ``errors.summary_json``. ``code_line``
148
+ (``path:line -> str | None``) and ``code_url`` (``path:line -> url | None``) add the
149
+ source line and a link to it."""
150
+ bold = (lambda s: f"*{s}*") if slack else (lambda s: s)
151
+ head = f"{':red_circle: ' if slack else ''}{bold(alert.name)}"
152
+ if alert.summary:
153
+ head += f": {alert.summary}"
154
+ head += f" (firing since {_when(alert.starts_at)})"
155
+ groups = summary["groups"]
156
+ total, new = summary["events"], sum(1 for g in groups if g["new"])
157
+ line2 = f"{total:,} error(s) in {summary['groups_total']} group(s)"
158
+ if new:
159
+ line2 += f", {bold(f'{new} new')}"
160
+ if summary.get("truncated"):
161
+ line2 += " (capped: counts are a lower bound)"
162
+ if alert.url or alert.dashboard_url:
163
+ line2 += " · " + _link("Open in Grafana", alert.dashboard_url or alert.url, slack)
164
+ if alert.silence_url and slack:
165
+ line2 += " · " + _link("Silence", alert.silence_url, slack)
166
+ out = [head, line2]
167
+ if not groups:
168
+ out.append("No error logs in this window: the alert isn't about logged errors.")
169
+ for i, g in enumerate(groups, 1):
170
+ trend = "new" if g["new"] else (g["trend"] or "")
171
+ out += [
172
+ "",
173
+ f"{i}. {bold(g['title'][:150])} · {g['count']:,}"
174
+ + (f" · {bold(trend)}" if trend else ""),
175
+ ]
176
+ causes = g["causes"]
177
+ if causes:
178
+ origin = f" ({g['origin']})" if g.get("origin") else ""
179
+ out.append(f" root cause: {causes[0][:150]}{origin}")
180
+ if g["raised_at"]:
181
+ code = code_line(g["raised_at"]) if code_line else None
182
+ url = code_url(g["raised_at"]) if code_url else None
183
+ text = f" raised: {_link(g['raised_at'], url, slack)}"
184
+ if code:
185
+ text += f" `{code[:100]}`" if slack else f" {code[:100]}"
186
+ out.append(text)
187
+ if g["logged_at"]:
188
+ url = code_url(g["logged_at"]) if code_url else None
189
+ out.append(f" logged: {_link(g['logged_at'], url, slack)}")
190
+ trace = (g["sample"].get("ids") or {}).get(trace_field) or (
191
+ g["sample"].get("ids") or {}
192
+ ).get("trace_id")
193
+ if trace:
194
+ out.append(f" example: `logsite trace {trace}`")
195
+ rest = summary["groups_total"] - len(groups)
196
+ if rest > 0:
197
+ out += ["", f"+ {rest} more group(s): `logsite errors`"]
198
+ return "\n".join(out)
199
+
200
+
201
+ def post_json(url: str, body: dict, timeout: float = 10.0) -> None:
202
+ req = urllib.request.Request(
203
+ url,
204
+ data=json.dumps(body).encode(),
205
+ headers={"Content-Type": "application/json"},
206
+ method="POST",
207
+ )
208
+ with urllib.request.urlopen(req, timeout=timeout) as resp:
209
+ resp.read()
210
+
211
+
212
+ # --- the server -------------------------------------------------------------------------------
213
+
214
+
215
+ class Enricher:
216
+ """Turns alerts into summaries. ``summarize(start, end, selector_labels) -> dict``
217
+ builds a summary (``errors.summary_json``) for a window, optionally narrowed by labels."""
218
+
219
+ def __init__(self, cfg: AlertsConfig, summarize, *, code_line=None, code_url=None,
220
+ trace_field: str = "trace_id", send=post_json, now=None) -> None: # fmt: skip
221
+ self.cfg = cfg
222
+ self.summarize = summarize
223
+ self.code_line, self.code_url = code_line, code_url
224
+ self.trace_field = trace_field
225
+ self.send = send
226
+ self.now = now or (lambda: datetime.now(timezone.utc))
227
+
228
+ def with_links(self, summary: dict) -> dict:
229
+ """Add each group's code links and a short trend, for a dashboard table."""
230
+ for g in summary.get("groups", []):
231
+ for key in ("raised_at", "logged_at"):
232
+ where = g.get(key)
233
+ g[key.replace("_at", "_url")] = (
234
+ self.code_url(where) if where and self.code_url else None
235
+ )
236
+ g[key.replace("_at", "_code")] = (
237
+ self.code_line(where) if where and self.code_line else None
238
+ )
239
+ # "OrderRepository.java:87": short enough for a table column
240
+ g[key.replace("_at", "_name")] = where.rsplit("/", 1)[-1] if where else None
241
+ g["trend_label"] = "new" if g.get("new") else (g.get("trend") or "")
242
+ # "OrderProcessingException: ..." without its package, for narrow columns
243
+ g["title_short"] = _PACKAGE.sub("", g.get("title") or "")
244
+ g["root_cause_short"] = _PACKAGE.sub("", g.get("root_cause") or "") or None
245
+ return summary
246
+
247
+ def handle(self, payload: Mapping[str, Any]) -> list[dict]:
248
+ results = []
249
+ for alert in parse_grafana(payload):
250
+ if alert.status != "firing":
251
+ results.append({"alert": alert.name, "status": alert.status, "skipped": True})
252
+ continue
253
+ start, end = window_for(alert, self.cfg.lookback_minutes, self.now())
254
+ labels = {k: v for k, v in alert.labels.items() if k in self.cfg.selector_labels}
255
+ summary = self.summarize(start, end, labels)
256
+ summary["groups"] = summary["groups"][: self.cfg.top]
257
+ kw = {
258
+ "code_line": self.code_line,
259
+ "code_url": self.code_url,
260
+ "trace_field": self.trace_field,
261
+ }
262
+ text = render(alert, summary, slack=False, **kw)
263
+ sent = []
264
+ if self.cfg.slack_webhook:
265
+ self.send(
266
+ self.cfg.slack_webhook, {"text": render(alert, summary, slack=True, **kw)}
267
+ )
268
+ sent.append("slack")
269
+ if self.cfg.webhook:
270
+ self.send(
271
+ self.cfg.webhook,
272
+ {"alert": alert.name, "labels": alert.labels, "text": text, "summary": summary},
273
+ )
274
+ sent.append("webhook")
275
+ results.append(
276
+ {
277
+ "alert": alert.name,
278
+ "status": "firing",
279
+ "sent": sent,
280
+ "text": text,
281
+ "summary": summary,
282
+ }
283
+ )
284
+ return results
285
+
286
+
287
+ def make_server(host: str, port: int, enricher: Enricher, token: str | None) -> ThreadingHTTPServer:
288
+ class Handler(BaseHTTPRequestHandler):
289
+ server_version = "logsite"
290
+
291
+ def log_message(self, fmt, *args): # quieter than the default stderr line per request
292
+ log.info("%s %s", self.address_string(), fmt % args)
293
+
294
+ def _authorized(self) -> bool:
295
+ if not token:
296
+ return True
297
+ header = self.headers.get("Authorization", "")
298
+ given = header.removeprefix("Bearer ").strip() if header.startswith("Bearer ") else None
299
+ if given is None:
300
+ given = (parse_qs(urlparse(self.path).query).get("token") or [None])[0]
301
+ return given is not None and secrets.compare_digest(given, token)
302
+
303
+ def _reply(self, status: int, body: Any) -> None:
304
+ data = json.dumps(body, indent=2).encode()
305
+ self.send_response(status)
306
+ self.send_header("Content-Type", "application/json")
307
+ self.send_header("Content-Length", str(len(data)))
308
+ self.end_headers()
309
+ self.wfile.write(data)
310
+
311
+ def do_GET(self): # noqa: N802
312
+ url = urlparse(self.path)
313
+ if url.path == "/healthz":
314
+ return self._reply(200, {"ok": True})
315
+ if not self._authorized():
316
+ return self._reply(401, {"error": "missing or wrong token"})
317
+ if url.path != "/errors":
318
+ return self._reply(404, {"error": "not found"})
319
+ q = parse_qs(url.query)
320
+ try:
321
+ end = _time(q.get("to", [None])[0]) or datetime.now(timezone.utc)
322
+ start = _time(q.get("from", [None])[0]) or end - timedelta(hours=1)
323
+ except ValueError as exc:
324
+ return self._reply(400, {"error": str(exc)})
325
+ labels = {
326
+ k: v[0] for k, v in q.items() if k in enricher.cfg.selector_labels and v[0]
327
+ } # an empty dashboard variable means "all"
328
+ try:
329
+ self._reply(200, enricher.with_links(enricher.summarize(start, end, labels)))
330
+ except Exception as exc: # the backend is down: say so, keep serving
331
+ self._reply(502, {"error": f"couldn't query the backend: {exc}"})
332
+
333
+ def do_POST(self): # noqa: N802
334
+ if urlparse(self.path).path not in ("/grafana/alert", "/alert"):
335
+ return self._reply(404, {"error": "not found"})
336
+ if not self._authorized():
337
+ return self._reply(401, {"error": "missing or wrong token"})
338
+ length = int(self.headers.get("Content-Length") or 0)
339
+ if length > MAX_BODY:
340
+ return self._reply(413, {"error": "payload too large"})
341
+ try:
342
+ payload = json.loads(self.rfile.read(length) or b"{}")
343
+ except json.JSONDecodeError:
344
+ return self._reply(400, {"error": "body is not JSON"})
345
+ try:
346
+ results = enricher.handle(payload)
347
+ except Exception as exc:
348
+ log.exception("couldn't enrich the alert")
349
+ return self._reply(502, {"error": f"couldn't build the summary: {exc}"})
350
+ self._reply(200, {"alerts": results})
351
+
352
+ return ThreadingHTTPServer((host, port), Handler)
353
+
354
+
355
+ def _time(value: str | None) -> datetime | None:
356
+ """A query time: ISO 8601, epoch seconds, or epoch milliseconds (Grafana's ${__from})."""
357
+ if not value:
358
+ return None
359
+ if value.isdigit():
360
+ n = int(value)
361
+ return datetime.fromtimestamp(n / 1000 if n > 10**11 else n, tz=timezone.utc)
362
+ ts = parse_iso(value)
363
+ return ts if ts.tzinfo else ts.replace(tzinfo=timezone.utc)
364
+
365
+
366
+ def with_selector(backend_cfg: dict, labels: Mapping[str, str], names: list[str]) -> dict:
367
+ """The backend config narrowed to the alert's labels (Loki only; others unchanged)."""
368
+ if backend_cfg.get("type") != "loki" or not labels:
369
+ return backend_cfg
370
+ return {**backend_cfg, "selector": narrow_selector(backend_cfg["selector"], labels, names)}
371
+
372
+
373
+ __all__ = [
374
+ "Alert",
375
+ "AlertsConfig",
376
+ "Enricher",
377
+ "make_server",
378
+ "narrow_selector",
379
+ "parse_grafana",
380
+ "render",
381
+ "with_selector",
382
+ ]
@@ -0,0 +1,73 @@
1
+ from .base import LogBackend, TimeRange, parse_line
2
+ from .cloudwatch import CloudWatchBackend
3
+ from .file import FileBackend
4
+ from .loki import LokiBackend
5
+ from .opensearch import OpenSearchBackend
6
+
7
+ __all__ = [
8
+ "CloudWatchBackend",
9
+ "FileBackend",
10
+ "LogBackend",
11
+ "LokiBackend",
12
+ "OpenSearchBackend",
13
+ "TimeRange",
14
+ "parse_line",
15
+ "from_config",
16
+ ]
17
+
18
+
19
+ def from_config(cfg: dict) -> LogBackend | None:
20
+ """Build a backend from the ``[backend]`` table of the logsite config.
21
+
22
+ Raises ValueError for a bad config and RuntimeError if the backend's extra isn't installed.
23
+ """
24
+ kind = cfg.get("type")
25
+ if not kind:
26
+ return None
27
+ try:
28
+ return _build(kind, cfg)
29
+ except KeyError as exc:
30
+ raise ValueError(f"backend type {kind!r} needs {exc.args[0]!r}") from None
31
+
32
+
33
+ def _build(kind: str, cfg: dict) -> LogBackend:
34
+ if kind == "file":
35
+ return FileBackend(cfg["path"])
36
+ if kind == "loki":
37
+ return LokiBackend(
38
+ cfg["url"],
39
+ cfg["selector"],
40
+ username=cfg.get("username"),
41
+ password=cfg.get("password"),
42
+ tenant_id=cfg.get("tenant_id"),
43
+ aggregate=bool(cfg.get("aggregate", False)),
44
+ parser=cfg.get("parser", "json"),
45
+ file_label=cfg.get("file_label", "pathname"),
46
+ line_label=cfg.get("line_label", "lineno"),
47
+ )
48
+ if kind in ("opensearch", "elasticsearch"):
49
+ return OpenSearchBackend(
50
+ cfg["url"],
51
+ cfg["index"],
52
+ query=cfg.get("query"),
53
+ timestamp_field=cfg.get("timestamp_field", "@timestamp"),
54
+ message_field=cfg.get("message_field", "message"),
55
+ username=cfg.get("username"),
56
+ password=cfg.get("password"),
57
+ api_key=cfg.get("api_key"),
58
+ aggregate=bool(cfg.get("aggregate", False)),
59
+ file_field=cfg.get("file_field", "log.origin.file.name"),
60
+ line_field=cfg.get("line_field", "log.origin.file.line"),
61
+ )
62
+ if kind == "cloudwatch":
63
+ return CloudWatchBackend(
64
+ cfg["log_group"],
65
+ region=cfg.get("region"),
66
+ profile=cfg.get("profile"),
67
+ filter_pattern=cfg.get("filter_pattern"),
68
+ log_stream_prefix=cfg.get("log_stream_prefix"),
69
+ )
70
+ raise ValueError(
71
+ f"unknown backend type: {kind!r} (supported: file, loki, opensearch, elasticsearch, "
72
+ "cloudwatch)"
73
+ )
@@ -0,0 +1,203 @@
1
+ """The one interface every backend implements. Nothing else in logsite knows the vendor."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ import re
7
+ from abc import ABC, abstractmethod
8
+ from collections import Counter
9
+ from collections.abc import Iterable, Mapping
10
+ from dataclasses import dataclass
11
+ from datetime import datetime, timedelta, timezone
12
+
13
+ from ..context import mentions
14
+ from ..matching import Matcher
15
+ from ..models import LogRecord, LogSite, Usage, parse_iso
16
+ from ..tracebacks import is_error
17
+
18
+
19
+ @dataclass(frozen=True)
20
+ class TimeRange:
21
+ start: datetime
22
+ end: datetime
23
+
24
+ @classmethod
25
+ def last(cls, hours: float = 24) -> TimeRange:
26
+ end = datetime.now(timezone.utc)
27
+ return cls(start=end - timedelta(hours=hours), end=end)
28
+
29
+ def previous(self) -> TimeRange:
30
+ """The range of the same length that ends where this one starts."""
31
+ return TimeRange(start=self.start - (self.end - self.start), end=self.start)
32
+
33
+
34
+ class LogBackend(ABC):
35
+ """Adapter for a log store (Loki, OpenSearch, CloudWatch, a local file, ...)."""
36
+
37
+ name: str = "base"
38
+
39
+ @abstractmethod
40
+ def query_logs(
41
+ self, time_range: TimeRange, contains: str | None = None, limit: int = 5000
42
+ ) -> list[LogRecord]:
43
+ """Return raw log records in ``time_range``, optionally filtered by substring."""
44
+
45
+ def count_by_line(
46
+ self, sites: Iterable[LogSite], time_range: TimeRange, limit: int = 5000
47
+ ) -> dict[str, int]:
48
+ """Return ``{site.id: count}`` for the given sites.
49
+
50
+ The default implementation fetches records and matches them client-side. Backends
51
+ that can aggregate server-side (e.g. on a ``lineno`` label) should override this.
52
+ """
53
+ return tally(Matcher(sites), {}, self.query_logs(time_range, limit=limit))
54
+
55
+ def find_records(
56
+ self, field: str, value: str, time_range: TimeRange, limit: int = 5000
57
+ ) -> list[LogRecord]:
58
+ """Records whose ``field`` (or a synonym, e.g. ``traceId`` for ``trace_id``) equals
59
+ ``value``; records without the field count if their message mentions it.
60
+
61
+ The default narrows with the backend's text filter (``contains``) and checks fields
62
+ client-side, which works for every backend whose stored line includes the value.
63
+ """
64
+ records = self.query_logs(time_range, contains=value, limit=limit)
65
+ return [r for r in records if mentions(r, field, value)]
66
+
67
+ def query_errors(self, time_range: TimeRange, limit: int = 5000) -> list[LogRecord]:
68
+ """Error records (error level, or carrying an exception) in ``time_range``.
69
+
70
+ The default filters :meth:`query_logs` client-side, so other lines count towards
71
+ ``limit``. Backends that can filter server-side override this.
72
+ """
73
+ return [r for r in self.query_logs(time_range, limit=limit) if is_error(r)]
74
+
75
+ def usage_by_line(
76
+ self, sites: Iterable[LogSite], time_range: TimeRange, limit: int = 5000
77
+ ) -> dict[str, Usage]:
78
+ """Return ``{site.id: Usage(count, bytes)}``.
79
+
80
+ The default measures the records it fetches, so it's exact but shares the ``limit``
81
+ cap of :meth:`count_by_line`. Backends that aggregate server-side override this.
82
+ """
83
+ return measure(Matcher(sites), self.query_logs(time_range, limit=limit))
84
+
85
+
86
+ def measure(matcher: Matcher, records: Iterable[LogRecord]) -> dict[str, Usage]:
87
+ """Count and sum the stored size of records per matched site."""
88
+ usage: dict[str, Usage] = {}
89
+ for record in records:
90
+ site = matcher.match(record)
91
+ if site is None:
92
+ continue
93
+ u = usage.setdefault(site.id, Usage())
94
+ u.count += 1
95
+ u.bytes += record.size if record.size is not None else len(record.message.encode())
96
+ return usage
97
+
98
+
99
+ def tally(
100
+ matcher: Matcher,
101
+ locations: Mapping[tuple[str, int], int],
102
+ records: Iterable[LogRecord] = (),
103
+ ) -> dict[str, int]:
104
+ """Turn server-side ``{(file, line): count}`` plus raw records into ``{site.id: count}``."""
105
+ counts: Counter[str] = Counter()
106
+ for (src, line), n in locations.items():
107
+ site = matcher.match_location(src, line)
108
+ if site is not None:
109
+ counts[site.id] += n
110
+ for record in records:
111
+ site = matcher.match(record)
112
+ if site is not None:
113
+ counts[site.id] += 1
114
+ return dict(counts)
115
+
116
+
117
+ # Where JSON loggers put the time: Python, logstash-logback-encoder / ECS (@timestamp), Go's
118
+ # slog / zerolog (time), zap (ts).
119
+ TIME_KEYS = ("timestamp", "@timestamp", "time", "ts", "asctime", "datetime")
120
+
121
+
122
+ def parse_line(raw: str, timestamp: datetime | None = None) -> LogRecord:
123
+ """Parse a log line: JSON objects become structured records, anything else is plain text."""
124
+ raw = raw.rstrip("\n")
125
+ stripped = raw.lstrip()
126
+ if stripped.startswith("{"):
127
+ try:
128
+ data = json.loads(stripped)
129
+ except json.JSONDecodeError:
130
+ data = None
131
+ if isinstance(data, dict) and isinstance(data.get("record"), dict):
132
+ # The traceback is only in "text", loguru's formatted line.
133
+ return _loguru_record({**data["record"], "_text": data.get("text")}, raw, timestamp)
134
+ if isinstance(data, dict):
135
+ message = next(
136
+ (str(data[k]) for k in ("message", "msg", "event", "log") if k in data), raw
137
+ )
138
+ ts = timestamp or _parse_ts(next((data[k] for k in TIME_KEYS if data.get(k)), None))
139
+ return LogRecord(timestamp=ts, message=message, fields=data, size=_size(raw))
140
+ return LogRecord(timestamp=timestamp or _leading_ts(raw), message=raw, size=_size(raw))
141
+
142
+
143
+ # "2026-10-14 10:00:05,123 ERROR ...": logging's default asctime, or ISO 8601.
144
+ _LEADING_TS = re.compile(
145
+ r"^\[?(\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}(?:[.,]\d{3}(?:\d{3})?)?(?:Z|[+-]\d{2}:\d{2})?)"
146
+ )
147
+
148
+
149
+ def _leading_ts(raw: str) -> datetime | None:
150
+ """The timestamp a plain-text line starts with, if any (naive means UTC)."""
151
+ m = _LEADING_TS.match(raw)
152
+ if not m:
153
+ return None
154
+ try:
155
+ return parse_iso(m.group(1).replace(",", "."))
156
+ except ValueError:
157
+ return None
158
+
159
+
160
+ def _loguru_record(rec: dict, raw: str, timestamp: datetime | None) -> LogRecord:
161
+ """loguru's ``serialize=True`` output: ``{"text": ..., "record": {"message", "file",
162
+ "line", "level", "extra", "time", ...}}``. Flattened so file/line matching and context
163
+ checks see the same fields as any other JSON log."""
164
+ file = rec.get("file") if isinstance(rec.get("file"), dict) else {}
165
+ level = rec.get("level") if isinstance(rec.get("level"), dict) else {}
166
+ time = rec.get("time") if isinstance(rec.get("time"), dict) else {}
167
+ extra = rec.get("extra") if isinstance(rec.get("extra"), dict) else {}
168
+ exception = rec.get("exception") if isinstance(rec.get("exception"), dict) else {}
169
+ text = rec.get("_text") if isinstance(rec.get("_text"), str) else ""
170
+ fields = {
171
+ **extra,
172
+ "exc_type": exception.get("type"),
173
+ "exc_message": exception.get("value"),
174
+ "exc_text": text if "Traceback (most recent call last):" in text else None,
175
+ "pathname": file.get("path") or file.get("name"),
176
+ "filename": file.get("name"),
177
+ "lineno": rec.get("line"),
178
+ "levelname": level.get("name"),
179
+ "funcName": rec.get("function"),
180
+ "name": rec.get("name"),
181
+ }
182
+ ts = timestamp or _parse_ts(time.get("timestamp") or time.get("repr"))
183
+ return LogRecord(
184
+ timestamp=ts,
185
+ message=str(rec.get("message", "")),
186
+ fields={k: v for k, v in fields.items() if v is not None},
187
+ size=_size(raw),
188
+ )
189
+
190
+
191
+ def _size(raw: str) -> int:
192
+ return len(raw.encode("utf-8", errors="replace"))
193
+
194
+
195
+ def _parse_ts(value: object) -> datetime | None:
196
+ if isinstance(value, (int, float)):
197
+ return datetime.fromtimestamp(value, tz=timezone.utc)
198
+ if isinstance(value, str):
199
+ try:
200
+ return parse_iso(value)
201
+ except ValueError:
202
+ return None
203
+ return None