logsite 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- logsite/__init__.py +18 -0
- logsite/__main__.py +3 -0
- logsite/alerts.py +382 -0
- logsite/backends/__init__.py +73 -0
- logsite/backends/base.py +203 -0
- logsite/backends/cloudwatch.py +105 -0
- logsite/backends/file.py +51 -0
- logsite/backends/loki.py +175 -0
- logsite/backends/opensearch.py +237 -0
- logsite/cli.py +980 -0
- logsite/config.py +180 -0
- logsite/context.py +228 -0
- logsite/cost.py +96 -0
- logsite/deploys.py +137 -0
- logsite/errors.py +408 -0
- logsite/fixes.py +161 -0
- logsite/git.py +201 -0
- logsite/grafana.py +592 -0
- logsite/index.py +408 -0
- logsite/lang/__init__.py +35 -0
- logsite/lang/base.py +61 -0
- logsite/lang/go.py +428 -0
- logsite/lang/java.py +312 -0
- logsite/lang/rules.py +110 -0
- logsite/leaks.py +257 -0
- logsite/lint.py +580 -0
- logsite/matching.py +91 -0
- logsite/mcp_server.py +465 -0
- logsite/models.py +171 -0
- logsite/pipeline.py +327 -0
- logsite/report.py +280 -0
- logsite/server.py +442 -0
- logsite/structure.py +72 -0
- logsite/term.py +76 -0
- logsite/timeline.py +135 -0
- logsite/tracebacks.py +425 -0
- logsite-0.1.0.dist-info/METADATA +762 -0
- logsite-0.1.0.dist-info/RECORD +41 -0
- logsite-0.1.0.dist-info/WHEEL +4 -0
- logsite-0.1.0.dist-info/entry_points.txt +2 -0
- logsite-0.1.0.dist-info/licenses/LICENSE +21 -0
logsite/__init__.py
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"""logsite: production log volume and log-quality checks, inline next to your log calls."""
|
|
2
|
+
|
|
3
|
+
from .index import index_project, index_source
|
|
4
|
+
from .lint import LintConfig, lint_source
|
|
5
|
+
from .matching import Matcher
|
|
6
|
+
from .models import Diagnostic, LogRecord, LogSite
|
|
7
|
+
|
|
8
|
+
__all__ = [
|
|
9
|
+
"Diagnostic",
|
|
10
|
+
"LintConfig",
|
|
11
|
+
"LogRecord",
|
|
12
|
+
"LogSite",
|
|
13
|
+
"Matcher",
|
|
14
|
+
"index_project",
|
|
15
|
+
"index_source",
|
|
16
|
+
"lint_source",
|
|
17
|
+
]
|
|
18
|
+
__version__ = "0.1.0"
|
logsite/__main__.py
ADDED
logsite/alerts.py
ADDED
|
@@ -0,0 +1,382 @@
|
|
|
1
|
+
"""Alert enrichment: when Grafana fires an alert, send the error summary with it.
|
|
2
|
+
|
|
3
|
+
``logsite alert-server`` receives Grafana's webhook contact point, summarizes the errors in
|
|
4
|
+
the alert's window (grouped by cause, with the line of code that raised them), and posts it
|
|
5
|
+
to Slack or another webhook. ``GET /errors`` serves the same summary as JSON.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
import logging
|
|
12
|
+
import os
|
|
13
|
+
import re
|
|
14
|
+
import secrets
|
|
15
|
+
import urllib.request
|
|
16
|
+
from collections.abc import Mapping
|
|
17
|
+
from dataclasses import dataclass, field
|
|
18
|
+
from datetime import datetime, timedelta, timezone
|
|
19
|
+
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
20
|
+
from typing import Any
|
|
21
|
+
from urllib.parse import parse_qs, urlparse
|
|
22
|
+
|
|
23
|
+
from .models import parse_iso
|
|
24
|
+
|
|
25
|
+
log = logging.getLogger(__name__)
|
|
26
|
+
|
|
27
|
+
MAX_BODY = 5 * 1024 * 1024 # Grafana batches alerts; this is plenty
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@dataclass
|
|
31
|
+
class AlertsConfig:
|
|
32
|
+
slack_webhook: str | None = None # Slack incoming webhook URL
|
|
33
|
+
webhook: str | None = None # any URL: gets the summary as JSON
|
|
34
|
+
token: str | None = None # required as "Authorization: Bearer <token>" or ?token=
|
|
35
|
+
lookback_minutes: float = 30.0 # summarize from this long before the alert started
|
|
36
|
+
top: int = 3 # error groups per alert
|
|
37
|
+
selector_labels: list[str] = field(default_factory=lambda: ["service_name"])
|
|
38
|
+
|
|
39
|
+
@classmethod
|
|
40
|
+
def from_dict(cls, data: Mapping) -> AlertsConfig:
|
|
41
|
+
known = set(cls.__dataclass_fields__) | {
|
|
42
|
+
f"{k}_env" for k in ("slack_webhook", "webhook", "token")
|
|
43
|
+
}
|
|
44
|
+
unknown = set(data) - known
|
|
45
|
+
if unknown:
|
|
46
|
+
raise ValueError(f"unknown [alerts] key(s) {sorted(unknown)}")
|
|
47
|
+
values = {k: v for k, v in data.items() if not k.endswith("_env")}
|
|
48
|
+
for key in ("slack_webhook", "webhook", "token"):
|
|
49
|
+
env = data.get(f"{key}_env")
|
|
50
|
+
if env and values.get(key) is None:
|
|
51
|
+
values[key] = os.environ.get(env)
|
|
52
|
+
cfg = cls(**values)
|
|
53
|
+
cfg.selector_labels = [str(x) for x in cfg.selector_labels]
|
|
54
|
+
return cfg
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
@dataclass
|
|
58
|
+
class Alert:
|
|
59
|
+
name: str
|
|
60
|
+
status: str # firing | resolved
|
|
61
|
+
labels: dict[str, str]
|
|
62
|
+
starts_at: datetime | None
|
|
63
|
+
url: str | None # the alert rule in Grafana
|
|
64
|
+
dashboard_url: str | None
|
|
65
|
+
summary: str | None # the rule's summary / description annotation
|
|
66
|
+
silence_url: str | None = None
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def parse_grafana(payload: Mapping[str, Any]) -> list[Alert]:
|
|
70
|
+
"""The alerts in a Grafana (unified alerting) or Alertmanager webhook payload."""
|
|
71
|
+
alerts = []
|
|
72
|
+
for a in payload.get("alerts") or []:
|
|
73
|
+
if not isinstance(a, dict):
|
|
74
|
+
continue
|
|
75
|
+
labels = {str(k): str(v) for k, v in (a.get("labels") or {}).items()}
|
|
76
|
+
annotations = a.get("annotations") or {}
|
|
77
|
+
started = a.get("startsAt")
|
|
78
|
+
try:
|
|
79
|
+
starts_at = parse_iso(started) if started and not started.startswith("0001") else None
|
|
80
|
+
except ValueError:
|
|
81
|
+
starts_at = None
|
|
82
|
+
alerts.append(
|
|
83
|
+
Alert(
|
|
84
|
+
name=labels.get("alertname", "alert"),
|
|
85
|
+
status=str(a.get("status", payload.get("status", "firing"))),
|
|
86
|
+
labels=labels,
|
|
87
|
+
starts_at=starts_at,
|
|
88
|
+
url=a.get("generatorURL") or None,
|
|
89
|
+
dashboard_url=a.get("panelURL") or a.get("dashboardURL") or None,
|
|
90
|
+
summary=annotations.get("summary") or annotations.get("description"),
|
|
91
|
+
silence_url=a.get("silenceURL") or None,
|
|
92
|
+
)
|
|
93
|
+
)
|
|
94
|
+
return alerts
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
_PACKAGE = re.compile(r"^(?:[a-z_$][\w$]*\.)+(?=[A-Z])") # "com.example.orders." in a class name
|
|
98
|
+
_LABEL_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def narrow_selector(selector: str, labels: Mapping[str, str], names: list[str]) -> str:
|
|
102
|
+
"""Add the alert's labels (those in ``names``) to a LogQL stream selector, so the
|
|
103
|
+
summary covers the service that alerted: ``{env="prod"}`` + service_name=checkout ->
|
|
104
|
+
``{env="prod", service_name="checkout"}``. Labels already in the selector are kept."""
|
|
105
|
+
body = selector.strip()
|
|
106
|
+
if not (body.startswith("{") and body.endswith("}")):
|
|
107
|
+
return selector
|
|
108
|
+
inner = body[1:-1].strip()
|
|
109
|
+
present = set(re.findall(r"([A-Za-z_][A-Za-z0-9_]*)\s*(?:=~|!=|!~|=)", inner))
|
|
110
|
+
extra = [
|
|
111
|
+
f"{n}={json.dumps(labels[n])}"
|
|
112
|
+
for n in names
|
|
113
|
+
if n in labels and n not in present and _LABEL_NAME.match(n)
|
|
114
|
+
]
|
|
115
|
+
if not extra:
|
|
116
|
+
return selector
|
|
117
|
+
return "{" + ", ".join([inner, *extra] if inner else extra) + "}"
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def window_for(alert: Alert, lookback_minutes: float, now: datetime) -> tuple[datetime, datetime]:
|
|
121
|
+
start = (alert.starts_at or now) - timedelta(minutes=lookback_minutes)
|
|
122
|
+
return min(start, now - timedelta(minutes=1)), now
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
# --- messages ---------------------------------------------------------------------------------
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _when(ts: datetime | None) -> str:
|
|
129
|
+
return ts.astimezone(timezone.utc).strftime("%H:%M UTC") if ts else "?"
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _link(text: str, url: str | None, slack: bool) -> str:
|
|
133
|
+
if not url:
|
|
134
|
+
return f"`{text}`" if slack else text
|
|
135
|
+
return f"<{url}|{text}>" if slack else f"{text} ({url})"
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def render(
|
|
139
|
+
alert: Alert,
|
|
140
|
+
summary: dict,
|
|
141
|
+
*,
|
|
142
|
+
slack: bool,
|
|
143
|
+
code_line: Any = None,
|
|
144
|
+
code_url: Any = None,
|
|
145
|
+
trace_field: str = "trace_id",
|
|
146
|
+
) -> str:
|
|
147
|
+
"""The message for one alert: ``summary`` is ``errors.summary_json``. ``code_line``
|
|
148
|
+
(``path:line -> str | None``) and ``code_url`` (``path:line -> url | None``) add the
|
|
149
|
+
source line and a link to it."""
|
|
150
|
+
bold = (lambda s: f"*{s}*") if slack else (lambda s: s)
|
|
151
|
+
head = f"{':red_circle: ' if slack else ''}{bold(alert.name)}"
|
|
152
|
+
if alert.summary:
|
|
153
|
+
head += f": {alert.summary}"
|
|
154
|
+
head += f" (firing since {_when(alert.starts_at)})"
|
|
155
|
+
groups = summary["groups"]
|
|
156
|
+
total, new = summary["events"], sum(1 for g in groups if g["new"])
|
|
157
|
+
line2 = f"{total:,} error(s) in {summary['groups_total']} group(s)"
|
|
158
|
+
if new:
|
|
159
|
+
line2 += f", {bold(f'{new} new')}"
|
|
160
|
+
if summary.get("truncated"):
|
|
161
|
+
line2 += " (capped: counts are a lower bound)"
|
|
162
|
+
if alert.url or alert.dashboard_url:
|
|
163
|
+
line2 += " · " + _link("Open in Grafana", alert.dashboard_url or alert.url, slack)
|
|
164
|
+
if alert.silence_url and slack:
|
|
165
|
+
line2 += " · " + _link("Silence", alert.silence_url, slack)
|
|
166
|
+
out = [head, line2]
|
|
167
|
+
if not groups:
|
|
168
|
+
out.append("No error logs in this window: the alert isn't about logged errors.")
|
|
169
|
+
for i, g in enumerate(groups, 1):
|
|
170
|
+
trend = "new" if g["new"] else (g["trend"] or "")
|
|
171
|
+
out += [
|
|
172
|
+
"",
|
|
173
|
+
f"{i}. {bold(g['title'][:150])} · {g['count']:,}"
|
|
174
|
+
+ (f" · {bold(trend)}" if trend else ""),
|
|
175
|
+
]
|
|
176
|
+
causes = g["causes"]
|
|
177
|
+
if causes:
|
|
178
|
+
origin = f" ({g['origin']})" if g.get("origin") else ""
|
|
179
|
+
out.append(f" root cause: {causes[0][:150]}{origin}")
|
|
180
|
+
if g["raised_at"]:
|
|
181
|
+
code = code_line(g["raised_at"]) if code_line else None
|
|
182
|
+
url = code_url(g["raised_at"]) if code_url else None
|
|
183
|
+
text = f" raised: {_link(g['raised_at'], url, slack)}"
|
|
184
|
+
if code:
|
|
185
|
+
text += f" `{code[:100]}`" if slack else f" {code[:100]}"
|
|
186
|
+
out.append(text)
|
|
187
|
+
if g["logged_at"]:
|
|
188
|
+
url = code_url(g["logged_at"]) if code_url else None
|
|
189
|
+
out.append(f" logged: {_link(g['logged_at'], url, slack)}")
|
|
190
|
+
trace = (g["sample"].get("ids") or {}).get(trace_field) or (
|
|
191
|
+
g["sample"].get("ids") or {}
|
|
192
|
+
).get("trace_id")
|
|
193
|
+
if trace:
|
|
194
|
+
out.append(f" example: `logsite trace {trace}`")
|
|
195
|
+
rest = summary["groups_total"] - len(groups)
|
|
196
|
+
if rest > 0:
|
|
197
|
+
out += ["", f"+ {rest} more group(s): `logsite errors`"]
|
|
198
|
+
return "\n".join(out)
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def post_json(url: str, body: dict, timeout: float = 10.0) -> None:
|
|
202
|
+
req = urllib.request.Request(
|
|
203
|
+
url,
|
|
204
|
+
data=json.dumps(body).encode(),
|
|
205
|
+
headers={"Content-Type": "application/json"},
|
|
206
|
+
method="POST",
|
|
207
|
+
)
|
|
208
|
+
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
|
209
|
+
resp.read()
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
# --- the server -------------------------------------------------------------------------------
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
class Enricher:
|
|
216
|
+
"""Turns alerts into summaries. ``summarize(start, end, selector_labels) -> dict``
|
|
217
|
+
builds a summary (``errors.summary_json``) for a window, optionally narrowed by labels."""
|
|
218
|
+
|
|
219
|
+
def __init__(self, cfg: AlertsConfig, summarize, *, code_line=None, code_url=None,
|
|
220
|
+
trace_field: str = "trace_id", send=post_json, now=None) -> None: # fmt: skip
|
|
221
|
+
self.cfg = cfg
|
|
222
|
+
self.summarize = summarize
|
|
223
|
+
self.code_line, self.code_url = code_line, code_url
|
|
224
|
+
self.trace_field = trace_field
|
|
225
|
+
self.send = send
|
|
226
|
+
self.now = now or (lambda: datetime.now(timezone.utc))
|
|
227
|
+
|
|
228
|
+
def with_links(self, summary: dict) -> dict:
|
|
229
|
+
"""Add each group's code links and a short trend, for a dashboard table."""
|
|
230
|
+
for g in summary.get("groups", []):
|
|
231
|
+
for key in ("raised_at", "logged_at"):
|
|
232
|
+
where = g.get(key)
|
|
233
|
+
g[key.replace("_at", "_url")] = (
|
|
234
|
+
self.code_url(where) if where and self.code_url else None
|
|
235
|
+
)
|
|
236
|
+
g[key.replace("_at", "_code")] = (
|
|
237
|
+
self.code_line(where) if where and self.code_line else None
|
|
238
|
+
)
|
|
239
|
+
# "OrderRepository.java:87": short enough for a table column
|
|
240
|
+
g[key.replace("_at", "_name")] = where.rsplit("/", 1)[-1] if where else None
|
|
241
|
+
g["trend_label"] = "new" if g.get("new") else (g.get("trend") or "")
|
|
242
|
+
# "OrderProcessingException: ..." without its package, for narrow columns
|
|
243
|
+
g["title_short"] = _PACKAGE.sub("", g.get("title") or "")
|
|
244
|
+
g["root_cause_short"] = _PACKAGE.sub("", g.get("root_cause") or "") or None
|
|
245
|
+
return summary
|
|
246
|
+
|
|
247
|
+
def handle(self, payload: Mapping[str, Any]) -> list[dict]:
|
|
248
|
+
results = []
|
|
249
|
+
for alert in parse_grafana(payload):
|
|
250
|
+
if alert.status != "firing":
|
|
251
|
+
results.append({"alert": alert.name, "status": alert.status, "skipped": True})
|
|
252
|
+
continue
|
|
253
|
+
start, end = window_for(alert, self.cfg.lookback_minutes, self.now())
|
|
254
|
+
labels = {k: v for k, v in alert.labels.items() if k in self.cfg.selector_labels}
|
|
255
|
+
summary = self.summarize(start, end, labels)
|
|
256
|
+
summary["groups"] = summary["groups"][: self.cfg.top]
|
|
257
|
+
kw = {
|
|
258
|
+
"code_line": self.code_line,
|
|
259
|
+
"code_url": self.code_url,
|
|
260
|
+
"trace_field": self.trace_field,
|
|
261
|
+
}
|
|
262
|
+
text = render(alert, summary, slack=False, **kw)
|
|
263
|
+
sent = []
|
|
264
|
+
if self.cfg.slack_webhook:
|
|
265
|
+
self.send(
|
|
266
|
+
self.cfg.slack_webhook, {"text": render(alert, summary, slack=True, **kw)}
|
|
267
|
+
)
|
|
268
|
+
sent.append("slack")
|
|
269
|
+
if self.cfg.webhook:
|
|
270
|
+
self.send(
|
|
271
|
+
self.cfg.webhook,
|
|
272
|
+
{"alert": alert.name, "labels": alert.labels, "text": text, "summary": summary},
|
|
273
|
+
)
|
|
274
|
+
sent.append("webhook")
|
|
275
|
+
results.append(
|
|
276
|
+
{
|
|
277
|
+
"alert": alert.name,
|
|
278
|
+
"status": "firing",
|
|
279
|
+
"sent": sent,
|
|
280
|
+
"text": text,
|
|
281
|
+
"summary": summary,
|
|
282
|
+
}
|
|
283
|
+
)
|
|
284
|
+
return results
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
def make_server(host: str, port: int, enricher: Enricher, token: str | None) -> ThreadingHTTPServer:
|
|
288
|
+
class Handler(BaseHTTPRequestHandler):
|
|
289
|
+
server_version = "logsite"
|
|
290
|
+
|
|
291
|
+
def log_message(self, fmt, *args): # quieter than the default stderr line per request
|
|
292
|
+
log.info("%s %s", self.address_string(), fmt % args)
|
|
293
|
+
|
|
294
|
+
def _authorized(self) -> bool:
|
|
295
|
+
if not token:
|
|
296
|
+
return True
|
|
297
|
+
header = self.headers.get("Authorization", "")
|
|
298
|
+
given = header.removeprefix("Bearer ").strip() if header.startswith("Bearer ") else None
|
|
299
|
+
if given is None:
|
|
300
|
+
given = (parse_qs(urlparse(self.path).query).get("token") or [None])[0]
|
|
301
|
+
return given is not None and secrets.compare_digest(given, token)
|
|
302
|
+
|
|
303
|
+
def _reply(self, status: int, body: Any) -> None:
|
|
304
|
+
data = json.dumps(body, indent=2).encode()
|
|
305
|
+
self.send_response(status)
|
|
306
|
+
self.send_header("Content-Type", "application/json")
|
|
307
|
+
self.send_header("Content-Length", str(len(data)))
|
|
308
|
+
self.end_headers()
|
|
309
|
+
self.wfile.write(data)
|
|
310
|
+
|
|
311
|
+
def do_GET(self): # noqa: N802
|
|
312
|
+
url = urlparse(self.path)
|
|
313
|
+
if url.path == "/healthz":
|
|
314
|
+
return self._reply(200, {"ok": True})
|
|
315
|
+
if not self._authorized():
|
|
316
|
+
return self._reply(401, {"error": "missing or wrong token"})
|
|
317
|
+
if url.path != "/errors":
|
|
318
|
+
return self._reply(404, {"error": "not found"})
|
|
319
|
+
q = parse_qs(url.query)
|
|
320
|
+
try:
|
|
321
|
+
end = _time(q.get("to", [None])[0]) or datetime.now(timezone.utc)
|
|
322
|
+
start = _time(q.get("from", [None])[0]) or end - timedelta(hours=1)
|
|
323
|
+
except ValueError as exc:
|
|
324
|
+
return self._reply(400, {"error": str(exc)})
|
|
325
|
+
labels = {
|
|
326
|
+
k: v[0] for k, v in q.items() if k in enricher.cfg.selector_labels and v[0]
|
|
327
|
+
} # an empty dashboard variable means "all"
|
|
328
|
+
try:
|
|
329
|
+
self._reply(200, enricher.with_links(enricher.summarize(start, end, labels)))
|
|
330
|
+
except Exception as exc: # the backend is down: say so, keep serving
|
|
331
|
+
self._reply(502, {"error": f"couldn't query the backend: {exc}"})
|
|
332
|
+
|
|
333
|
+
def do_POST(self): # noqa: N802
|
|
334
|
+
if urlparse(self.path).path not in ("/grafana/alert", "/alert"):
|
|
335
|
+
return self._reply(404, {"error": "not found"})
|
|
336
|
+
if not self._authorized():
|
|
337
|
+
return self._reply(401, {"error": "missing or wrong token"})
|
|
338
|
+
length = int(self.headers.get("Content-Length") or 0)
|
|
339
|
+
if length > MAX_BODY:
|
|
340
|
+
return self._reply(413, {"error": "payload too large"})
|
|
341
|
+
try:
|
|
342
|
+
payload = json.loads(self.rfile.read(length) or b"{}")
|
|
343
|
+
except json.JSONDecodeError:
|
|
344
|
+
return self._reply(400, {"error": "body is not JSON"})
|
|
345
|
+
try:
|
|
346
|
+
results = enricher.handle(payload)
|
|
347
|
+
except Exception as exc:
|
|
348
|
+
log.exception("couldn't enrich the alert")
|
|
349
|
+
return self._reply(502, {"error": f"couldn't build the summary: {exc}"})
|
|
350
|
+
self._reply(200, {"alerts": results})
|
|
351
|
+
|
|
352
|
+
return ThreadingHTTPServer((host, port), Handler)
|
|
353
|
+
|
|
354
|
+
|
|
355
|
+
def _time(value: str | None) -> datetime | None:
|
|
356
|
+
"""A query time: ISO 8601, epoch seconds, or epoch milliseconds (Grafana's ${__from})."""
|
|
357
|
+
if not value:
|
|
358
|
+
return None
|
|
359
|
+
if value.isdigit():
|
|
360
|
+
n = int(value)
|
|
361
|
+
return datetime.fromtimestamp(n / 1000 if n > 10**11 else n, tz=timezone.utc)
|
|
362
|
+
ts = parse_iso(value)
|
|
363
|
+
return ts if ts.tzinfo else ts.replace(tzinfo=timezone.utc)
|
|
364
|
+
|
|
365
|
+
|
|
366
|
+
def with_selector(backend_cfg: dict, labels: Mapping[str, str], names: list[str]) -> dict:
|
|
367
|
+
"""The backend config narrowed to the alert's labels (Loki only; others unchanged)."""
|
|
368
|
+
if backend_cfg.get("type") != "loki" or not labels:
|
|
369
|
+
return backend_cfg
|
|
370
|
+
return {**backend_cfg, "selector": narrow_selector(backend_cfg["selector"], labels, names)}
|
|
371
|
+
|
|
372
|
+
|
|
373
|
+
__all__ = [
|
|
374
|
+
"Alert",
|
|
375
|
+
"AlertsConfig",
|
|
376
|
+
"Enricher",
|
|
377
|
+
"make_server",
|
|
378
|
+
"narrow_selector",
|
|
379
|
+
"parse_grafana",
|
|
380
|
+
"render",
|
|
381
|
+
"with_selector",
|
|
382
|
+
]
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
from .base import LogBackend, TimeRange, parse_line
|
|
2
|
+
from .cloudwatch import CloudWatchBackend
|
|
3
|
+
from .file import FileBackend
|
|
4
|
+
from .loki import LokiBackend
|
|
5
|
+
from .opensearch import OpenSearchBackend
|
|
6
|
+
|
|
7
|
+
__all__ = [
|
|
8
|
+
"CloudWatchBackend",
|
|
9
|
+
"FileBackend",
|
|
10
|
+
"LogBackend",
|
|
11
|
+
"LokiBackend",
|
|
12
|
+
"OpenSearchBackend",
|
|
13
|
+
"TimeRange",
|
|
14
|
+
"parse_line",
|
|
15
|
+
"from_config",
|
|
16
|
+
]
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def from_config(cfg: dict) -> LogBackend | None:
|
|
20
|
+
"""Build a backend from the ``[backend]`` table of the logsite config.
|
|
21
|
+
|
|
22
|
+
Raises ValueError for a bad config and RuntimeError if the backend's extra isn't installed.
|
|
23
|
+
"""
|
|
24
|
+
kind = cfg.get("type")
|
|
25
|
+
if not kind:
|
|
26
|
+
return None
|
|
27
|
+
try:
|
|
28
|
+
return _build(kind, cfg)
|
|
29
|
+
except KeyError as exc:
|
|
30
|
+
raise ValueError(f"backend type {kind!r} needs {exc.args[0]!r}") from None
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _build(kind: str, cfg: dict) -> LogBackend:
|
|
34
|
+
if kind == "file":
|
|
35
|
+
return FileBackend(cfg["path"])
|
|
36
|
+
if kind == "loki":
|
|
37
|
+
return LokiBackend(
|
|
38
|
+
cfg["url"],
|
|
39
|
+
cfg["selector"],
|
|
40
|
+
username=cfg.get("username"),
|
|
41
|
+
password=cfg.get("password"),
|
|
42
|
+
tenant_id=cfg.get("tenant_id"),
|
|
43
|
+
aggregate=bool(cfg.get("aggregate", False)),
|
|
44
|
+
parser=cfg.get("parser", "json"),
|
|
45
|
+
file_label=cfg.get("file_label", "pathname"),
|
|
46
|
+
line_label=cfg.get("line_label", "lineno"),
|
|
47
|
+
)
|
|
48
|
+
if kind in ("opensearch", "elasticsearch"):
|
|
49
|
+
return OpenSearchBackend(
|
|
50
|
+
cfg["url"],
|
|
51
|
+
cfg["index"],
|
|
52
|
+
query=cfg.get("query"),
|
|
53
|
+
timestamp_field=cfg.get("timestamp_field", "@timestamp"),
|
|
54
|
+
message_field=cfg.get("message_field", "message"),
|
|
55
|
+
username=cfg.get("username"),
|
|
56
|
+
password=cfg.get("password"),
|
|
57
|
+
api_key=cfg.get("api_key"),
|
|
58
|
+
aggregate=bool(cfg.get("aggregate", False)),
|
|
59
|
+
file_field=cfg.get("file_field", "log.origin.file.name"),
|
|
60
|
+
line_field=cfg.get("line_field", "log.origin.file.line"),
|
|
61
|
+
)
|
|
62
|
+
if kind == "cloudwatch":
|
|
63
|
+
return CloudWatchBackend(
|
|
64
|
+
cfg["log_group"],
|
|
65
|
+
region=cfg.get("region"),
|
|
66
|
+
profile=cfg.get("profile"),
|
|
67
|
+
filter_pattern=cfg.get("filter_pattern"),
|
|
68
|
+
log_stream_prefix=cfg.get("log_stream_prefix"),
|
|
69
|
+
)
|
|
70
|
+
raise ValueError(
|
|
71
|
+
f"unknown backend type: {kind!r} (supported: file, loki, opensearch, elasticsearch, "
|
|
72
|
+
"cloudwatch)"
|
|
73
|
+
)
|
logsite/backends/base.py
ADDED
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
"""The one interface every backend implements. Nothing else in logsite knows the vendor."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import re
|
|
7
|
+
from abc import ABC, abstractmethod
|
|
8
|
+
from collections import Counter
|
|
9
|
+
from collections.abc import Iterable, Mapping
|
|
10
|
+
from dataclasses import dataclass
|
|
11
|
+
from datetime import datetime, timedelta, timezone
|
|
12
|
+
|
|
13
|
+
from ..context import mentions
|
|
14
|
+
from ..matching import Matcher
|
|
15
|
+
from ..models import LogRecord, LogSite, Usage, parse_iso
|
|
16
|
+
from ..tracebacks import is_error
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@dataclass(frozen=True)
|
|
20
|
+
class TimeRange:
|
|
21
|
+
start: datetime
|
|
22
|
+
end: datetime
|
|
23
|
+
|
|
24
|
+
@classmethod
|
|
25
|
+
def last(cls, hours: float = 24) -> TimeRange:
|
|
26
|
+
end = datetime.now(timezone.utc)
|
|
27
|
+
return cls(start=end - timedelta(hours=hours), end=end)
|
|
28
|
+
|
|
29
|
+
def previous(self) -> TimeRange:
|
|
30
|
+
"""The range of the same length that ends where this one starts."""
|
|
31
|
+
return TimeRange(start=self.start - (self.end - self.start), end=self.start)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class LogBackend(ABC):
|
|
35
|
+
"""Adapter for a log store (Loki, OpenSearch, CloudWatch, a local file, ...)."""
|
|
36
|
+
|
|
37
|
+
name: str = "base"
|
|
38
|
+
|
|
39
|
+
@abstractmethod
|
|
40
|
+
def query_logs(
|
|
41
|
+
self, time_range: TimeRange, contains: str | None = None, limit: int = 5000
|
|
42
|
+
) -> list[LogRecord]:
|
|
43
|
+
"""Return raw log records in ``time_range``, optionally filtered by substring."""
|
|
44
|
+
|
|
45
|
+
def count_by_line(
|
|
46
|
+
self, sites: Iterable[LogSite], time_range: TimeRange, limit: int = 5000
|
|
47
|
+
) -> dict[str, int]:
|
|
48
|
+
"""Return ``{site.id: count}`` for the given sites.
|
|
49
|
+
|
|
50
|
+
The default implementation fetches records and matches them client-side. Backends
|
|
51
|
+
that can aggregate server-side (e.g. on a ``lineno`` label) should override this.
|
|
52
|
+
"""
|
|
53
|
+
return tally(Matcher(sites), {}, self.query_logs(time_range, limit=limit))
|
|
54
|
+
|
|
55
|
+
def find_records(
|
|
56
|
+
self, field: str, value: str, time_range: TimeRange, limit: int = 5000
|
|
57
|
+
) -> list[LogRecord]:
|
|
58
|
+
"""Records whose ``field`` (or a synonym, e.g. ``traceId`` for ``trace_id``) equals
|
|
59
|
+
``value``; records without the field count if their message mentions it.
|
|
60
|
+
|
|
61
|
+
The default narrows with the backend's text filter (``contains``) and checks fields
|
|
62
|
+
client-side, which works for every backend whose stored line includes the value.
|
|
63
|
+
"""
|
|
64
|
+
records = self.query_logs(time_range, contains=value, limit=limit)
|
|
65
|
+
return [r for r in records if mentions(r, field, value)]
|
|
66
|
+
|
|
67
|
+
def query_errors(self, time_range: TimeRange, limit: int = 5000) -> list[LogRecord]:
|
|
68
|
+
"""Error records (error level, or carrying an exception) in ``time_range``.
|
|
69
|
+
|
|
70
|
+
The default filters :meth:`query_logs` client-side, so other lines count towards
|
|
71
|
+
``limit``. Backends that can filter server-side override this.
|
|
72
|
+
"""
|
|
73
|
+
return [r for r in self.query_logs(time_range, limit=limit) if is_error(r)]
|
|
74
|
+
|
|
75
|
+
def usage_by_line(
|
|
76
|
+
self, sites: Iterable[LogSite], time_range: TimeRange, limit: int = 5000
|
|
77
|
+
) -> dict[str, Usage]:
|
|
78
|
+
"""Return ``{site.id: Usage(count, bytes)}``.
|
|
79
|
+
|
|
80
|
+
The default measures the records it fetches, so it's exact but shares the ``limit``
|
|
81
|
+
cap of :meth:`count_by_line`. Backends that aggregate server-side override this.
|
|
82
|
+
"""
|
|
83
|
+
return measure(Matcher(sites), self.query_logs(time_range, limit=limit))
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def measure(matcher: Matcher, records: Iterable[LogRecord]) -> dict[str, Usage]:
|
|
87
|
+
"""Count and sum the stored size of records per matched site."""
|
|
88
|
+
usage: dict[str, Usage] = {}
|
|
89
|
+
for record in records:
|
|
90
|
+
site = matcher.match(record)
|
|
91
|
+
if site is None:
|
|
92
|
+
continue
|
|
93
|
+
u = usage.setdefault(site.id, Usage())
|
|
94
|
+
u.count += 1
|
|
95
|
+
u.bytes += record.size if record.size is not None else len(record.message.encode())
|
|
96
|
+
return usage
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def tally(
|
|
100
|
+
matcher: Matcher,
|
|
101
|
+
locations: Mapping[tuple[str, int], int],
|
|
102
|
+
records: Iterable[LogRecord] = (),
|
|
103
|
+
) -> dict[str, int]:
|
|
104
|
+
"""Turn server-side ``{(file, line): count}`` plus raw records into ``{site.id: count}``."""
|
|
105
|
+
counts: Counter[str] = Counter()
|
|
106
|
+
for (src, line), n in locations.items():
|
|
107
|
+
site = matcher.match_location(src, line)
|
|
108
|
+
if site is not None:
|
|
109
|
+
counts[site.id] += n
|
|
110
|
+
for record in records:
|
|
111
|
+
site = matcher.match(record)
|
|
112
|
+
if site is not None:
|
|
113
|
+
counts[site.id] += 1
|
|
114
|
+
return dict(counts)
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
# Where JSON loggers put the time: Python, logstash-logback-encoder / ECS (@timestamp), Go's
|
|
118
|
+
# slog / zerolog (time), zap (ts).
|
|
119
|
+
TIME_KEYS = ("timestamp", "@timestamp", "time", "ts", "asctime", "datetime")
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def parse_line(raw: str, timestamp: datetime | None = None) -> LogRecord:
|
|
123
|
+
"""Parse a log line: JSON objects become structured records, anything else is plain text."""
|
|
124
|
+
raw = raw.rstrip("\n")
|
|
125
|
+
stripped = raw.lstrip()
|
|
126
|
+
if stripped.startswith("{"):
|
|
127
|
+
try:
|
|
128
|
+
data = json.loads(stripped)
|
|
129
|
+
except json.JSONDecodeError:
|
|
130
|
+
data = None
|
|
131
|
+
if isinstance(data, dict) and isinstance(data.get("record"), dict):
|
|
132
|
+
# The traceback is only in "text", loguru's formatted line.
|
|
133
|
+
return _loguru_record({**data["record"], "_text": data.get("text")}, raw, timestamp)
|
|
134
|
+
if isinstance(data, dict):
|
|
135
|
+
message = next(
|
|
136
|
+
(str(data[k]) for k in ("message", "msg", "event", "log") if k in data), raw
|
|
137
|
+
)
|
|
138
|
+
ts = timestamp or _parse_ts(next((data[k] for k in TIME_KEYS if data.get(k)), None))
|
|
139
|
+
return LogRecord(timestamp=ts, message=message, fields=data, size=_size(raw))
|
|
140
|
+
return LogRecord(timestamp=timestamp or _leading_ts(raw), message=raw, size=_size(raw))
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
# "2026-10-14 10:00:05,123 ERROR ...": logging's default asctime, or ISO 8601.
|
|
144
|
+
_LEADING_TS = re.compile(
|
|
145
|
+
r"^\[?(\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}(?:[.,]\d{3}(?:\d{3})?)?(?:Z|[+-]\d{2}:\d{2})?)"
|
|
146
|
+
)
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def _leading_ts(raw: str) -> datetime | None:
|
|
150
|
+
"""The timestamp a plain-text line starts with, if any (naive means UTC)."""
|
|
151
|
+
m = _LEADING_TS.match(raw)
|
|
152
|
+
if not m:
|
|
153
|
+
return None
|
|
154
|
+
try:
|
|
155
|
+
return parse_iso(m.group(1).replace(",", "."))
|
|
156
|
+
except ValueError:
|
|
157
|
+
return None
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def _loguru_record(rec: dict, raw: str, timestamp: datetime | None) -> LogRecord:
|
|
161
|
+
"""loguru's ``serialize=True`` output: ``{"text": ..., "record": {"message", "file",
|
|
162
|
+
"line", "level", "extra", "time", ...}}``. Flattened so file/line matching and context
|
|
163
|
+
checks see the same fields as any other JSON log."""
|
|
164
|
+
file = rec.get("file") if isinstance(rec.get("file"), dict) else {}
|
|
165
|
+
level = rec.get("level") if isinstance(rec.get("level"), dict) else {}
|
|
166
|
+
time = rec.get("time") if isinstance(rec.get("time"), dict) else {}
|
|
167
|
+
extra = rec.get("extra") if isinstance(rec.get("extra"), dict) else {}
|
|
168
|
+
exception = rec.get("exception") if isinstance(rec.get("exception"), dict) else {}
|
|
169
|
+
text = rec.get("_text") if isinstance(rec.get("_text"), str) else ""
|
|
170
|
+
fields = {
|
|
171
|
+
**extra,
|
|
172
|
+
"exc_type": exception.get("type"),
|
|
173
|
+
"exc_message": exception.get("value"),
|
|
174
|
+
"exc_text": text if "Traceback (most recent call last):" in text else None,
|
|
175
|
+
"pathname": file.get("path") or file.get("name"),
|
|
176
|
+
"filename": file.get("name"),
|
|
177
|
+
"lineno": rec.get("line"),
|
|
178
|
+
"levelname": level.get("name"),
|
|
179
|
+
"funcName": rec.get("function"),
|
|
180
|
+
"name": rec.get("name"),
|
|
181
|
+
}
|
|
182
|
+
ts = timestamp or _parse_ts(time.get("timestamp") or time.get("repr"))
|
|
183
|
+
return LogRecord(
|
|
184
|
+
timestamp=ts,
|
|
185
|
+
message=str(rec.get("message", "")),
|
|
186
|
+
fields={k: v for k, v in fields.items() if v is not None},
|
|
187
|
+
size=_size(raw),
|
|
188
|
+
)
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def _size(raw: str) -> int:
|
|
192
|
+
return len(raw.encode("utf-8", errors="replace"))
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def _parse_ts(value: object) -> datetime | None:
|
|
196
|
+
if isinstance(value, (int, float)):
|
|
197
|
+
return datetime.fromtimestamp(value, tz=timezone.utc)
|
|
198
|
+
if isinstance(value, str):
|
|
199
|
+
try:
|
|
200
|
+
return parse_iso(value)
|
|
201
|
+
except ValueError:
|
|
202
|
+
return None
|
|
203
|
+
return None
|