stackdoctor 0.1.0__py3-none-any.whl

stackdoctor/__main__.py

@@ -0,0 +1,3 @@
+from .server import main
+
+main()

stackdoctor/checks/__init__.py

@@ -0,0 +1,29 @@
+"""Checks. Every check is a plain sync function returning a dict.
+
+Checks may include an `events` list: notable things with a timestamp, which
+diagnose() merges into one cross-system timeline.
+"""
+
+from __future__ import annotations
+
+import datetime as dt
+
+
+def utcnow() -> dt.datetime:
+    return dt.datetime.now(dt.timezone.utc)
+
+
+def to_utc(ts: dt.datetime | float | None) -> dt.datetime | None:
+    if ts is None:
+        return None
+    if isinstance(ts, (int, float)):
+        return dt.datetime.fromtimestamp(ts, dt.timezone.utc)
+    if ts.tzinfo is None:
+        ts = ts.astimezone()  # naive = local time
+    return ts.astimezone(dt.timezone.utc)
+
+
+def event(ts, source: str, kind: str, detail: str, severity: str = "info", **extra) -> dict:
+    """A timeline event. `kind` is a stable keyword used by the cause → effect rules."""
+    return {"ts": to_utc(ts) or utcnow(), "source": source, "kind": kind,
+            "severity": severity, "detail": detail, **extra}

stackdoctor/checks/celery.py

@@ -0,0 +1,319 @@
+"""Celery checks without Flower: the inspect API plus reading the broker directly.
+
+Only ping/active/reserved/active_queues/stats/query_task are used. Nothing here
+sends, revokes, retries or shuts down tasks or workers.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import datetime as dt
+import importlib
+import json
+import os
+import re
+import sys
+import threading
+import time
+
+from celery import Celery
+
+from ..config import get_config, skipped
+from ..safety import preview
+from . import event, to_utc
+from .redis import client as redis_client, glob_escape
+
+SOURCE = "celery"
+_TASK_ID_RE = re.compile(r"^[A-Za-z0-9_.:-]{1,128}$")
+_META_PREFIX = "celery-task-meta-"
+_HISTORY: dict[str, list[tuple[float, int]]] = {}  # queue -> recent (time, length) samples
+
+
+_app = None
+_app_lock = threading.Lock()  # diagnose() calls checks from several threads at once
+
+
+def get_app():
+    global _app
+    with _app_lock:
+        if _app is None:
+            _app = _load_app()
+        return _app
+
+
+def _load_app():
+    """Load CELERY_APP (`pkg.module:app` or `pkg.module`), else a bare app on the broker."""
+    cfg = get_config()
+    if cfg.celery_app:
+        if os.getcwd() not in sys.path:
+            sys.path.insert(0, os.getcwd())
+        mod_name, _, attr = cfg.celery_app.partition(":")
+        # Importing user code must never print to stdout: that is the MCP channel.
+        with contextlib.redirect_stdout(sys.stderr):
+            module = importlib.import_module(mod_name)
+        if attr:
+            return getattr(module, attr)
+        for name in ("app", "celery", "celery_app"):
+            if hasattr(module, name):
+                return getattr(module, name)
+        raise ImportError(f"No Celery app found in {mod_name} (use module:attr)")
+    if cfg.celery_broker_url:
+        return Celery("stackdoctor", broker=cfg.celery_broker_url,
+                      backend=cfg.celery_result_backend)
+    return None
+
+
+def _not_configured():
+    cfg = get_config()
+    if not (cfg.celery_app or cfg.celery_broker_url):
+        return skipped("CELERY_APP / CELERY_BROKER_URL not set")
+    return None
+
+
+def _inspect(app, destination=None):
+    timeout = get_config().celery_inspect_timeout_s
+    return app.control.inspect(timeout=timeout, destination=destination,
+                               limit=len(destination) if destination else None)
+
+
+def _task_summary(t: dict) -> dict:
+    started = t.get("time_start")
+    return {
+        "id": t.get("id"), "name": t.get("name"),
+        "args": preview(t.get("args"), 120), "kwargs": preview(t.get("kwargs"), 120),
+        "started": to_utc(started),
+        "runtime_s": round(time.time() - started, 1) if started else None,
+        "queue": (t.get("delivery_info") or {}).get("routing_key"),
+    }
+
+
+def workers() -> dict:
+    if (s := _not_configured()):
+        return s
+    cfg, app = get_config(), get_app()
+    ping = _inspect(app).ping() or {}
+    events = []
+    if not ping:
+        events.append(event(None, SOURCE, "no_workers",
+                            "No Celery workers replied to ping", "critical"))
+        return {"alive_count": 0, "workers": [], "events": events}
+
+    names = sorted(ping)
+    insp = _inspect(app, destination=names)
+    active, reserved = insp.active() or {}, insp.reserved() or {}
+    queues, stats = insp.active_queues() or {}, insp.stats() or {}
+
+    out = []
+    for name in names:
+        st = stats.get(name, {})
+        concurrency = (st.get("pool") or {}).get("max-concurrency")
+        act = [_task_summary(t) for t in active.get(name, [])]
+        res = [_task_summary(t) for t in reserved.get(name, [])]
+        out.append({
+            "name": name, "alive": True,
+            "queues": [q.get("name") for q in queues.get(name, [])],
+            "concurrency": concurrency,
+            "active_count": len(act), "reserved_count": len(res),
+            "active": act[:20], "reserved": res[:10],
+            "processed_total": sum((st.get("total") or {}).values()),
+        })
+        for t in act:
+            if t["runtime_s"] and t["runtime_s"] >= cfg.long_task_s:
+                events.append(event(t["started"], SOURCE, "task_long_running",
+                                    f"{t['name']}[{t['id']}] running {t['runtime_s']}s on {name}",
+                                    "warning", task_id=t["id"]))
+        if concurrency and len(act) >= concurrency:
+            events.append(event(None, SOURCE, "workers_saturated",
+                                f"{name}: all {concurrency} slots busy, {len(res)} reserved", "warning"))
+
+    if cfg.expected_workers and len(names) < cfg.expected_workers:
+        events.append(event(None, SOURCE, "worker_missing",
+                            f"Only {len(names)} of {cfg.expected_workers} expected workers replied",
+                            "critical"))
+    return {"alive_count": len(names), "workers": out, "events": events}
+
+
+def _queue_names(app, given: list[str] | None) -> list[str]:
+    if given:
+        return given
+    names = set(get_config().celery_queues)
+    names.add(app.conf.task_default_queue or "celery")
+    for q in app.conf.task_queues or []:
+        names.add(getattr(q, "name", q))
+    for route in (app.conf.task_routes or {}).values() if isinstance(app.conf.task_routes, dict) else []:
+        if isinstance(route, dict) and route.get("queue"):
+            names.add(route["queue"])
+    return sorted(names)
+
+
+def _redis_queue_lengths(app, url: str, names: list[str]) -> dict:
+    opts = app.conf.broker_transport_options or {}
+    steps = opts.get("priority_steps") or [0, 3, 6, 9]
+    sep = opts.get("sep", "\x06\x16")
+    prefix = opts.get("global_keyprefix", "")
+    r = redis_client(url)
+    result = {}
+    for q in names:
+        by_priority = {}
+        for step in steps:
+            key = prefix + (f"{q}{sep}{step}" if step else q)
+            n = r.cmd("LLEN", key)
+            if n:
+                by_priority[str(step)] = n
+        # Steps we don't know about (custom priority_steps on the producer side).
+        extra, _ = r.scan_iter(glob_escape(prefix + q + sep) + "*", 50)
+        for key in extra:
+            step = key.rsplit(sep, 1)[-1]
+            if step.isdigit() and int(step) not in steps:
+                by_priority[step] = r.cmd("LLEN", key)
+        result[q] = {"messages": sum(by_priority.values()),
+                     "by_priority": by_priority if len(by_priority) > 1 else None}
+    unacked = r.cmd("HLEN", prefix + "unacked")
+    return {"queues": result, "unacked_total": unacked}
+
+
+def _amqp_queue_lengths(app, names: list[str]) -> dict:
+    result = {}
+    for q in names:
+        with app.connection_for_read() as conn:
+            try:
+                # passive=True only checks the queue; it never creates anything.
+                _, messages, consumers = conn.default_channel.queue_declare(queue=q, passive=True)
+                result[q] = {"messages": messages, "consumers": consumers}
+            except conn.channel_errors:
+                result[q] = {"error": "queue does not exist"}
+    return {"queues": result}
+
+
+def queue_lengths(queues: list[str] | None = None) -> dict:
+    if (s := _not_configured()):
+        return s
+    cfg, app = get_config(), get_app()
+    url = app.conf.broker_url or cfg.celery_broker_url or ""
+    names = _queue_names(app, queues)
+    if url.startswith(("redis://", "rediss://", "unix://")):
+        out = _redis_queue_lengths(app, url, names)
+    elif url.startswith(("amqp://", "amqps://", "pyamqp://")):
+        out = _amqp_queue_lengths(app, names)
+    else:
+        return {"error": f"Queue lengths are supported for Redis and RabbitMQ brokers, not {url.split(':')[0]}"}
+
+    now, events = time.time(), []
+    for q, info in out["queues"].items():
+        n = info.get("messages")
+        if n is None:
+            continue
+        hist = _HISTORY.setdefault(q, [])
+        hist.append((now, n))
+        del hist[:-10]
+        older = [(t, m) for t, m in hist if now - t >= 5]
+        if older:
+            t0, m0 = older[-1]
+            info["change_since_last_check"] = {"seconds_ago": round(now - t0), "delta": n - m0}
+        if n >= cfg.queue_threshold:
+            events.append(event(None, SOURCE, "queue_backlog",
+                                f"Queue '{q}' has {n} waiting messages (threshold {cfg.queue_threshold})",
+                                "warning", queue=q, messages=n))
+        if info.get("consumers") == 0 and n:
+            events.append(event(None, SOURCE, "queue_no_consumer",
+                                f"Queue '{q}' has {n} messages and no consumers", "critical", queue=q))
+    out["events"] = events
+    return out
+
+
+def _backend(app) -> tuple[str | None, str | None]:
+    """Return (redis_url, None) if results are readable, else (None, reason)."""
+    cfg = get_config()
+    url = cfg.celery_result_backend or app.conf.result_backend
+    if not url:
+        return None, "no result backend is configured (CELERY_RESULT_BACKEND / app.conf.result_backend)"
+    if not str(url).startswith(("redis://", "rediss://")):
+        return None, f"the result backend is '{str(url).split(':')[0]}'; stackdoctor only reads Redis result backends"
+    return url, None
+
+
+def failed_tasks(limit: int = 20) -> dict:
+    if (s := _not_configured()):
+        return s
+    app = get_app()
+    url, reason = _backend(app)
+    if reason:
+        return {"visible": False, "reason": f"Task failures are not visible because {reason}. "
+                                             "Check logs for task errors instead."}
+    r = redis_client(url)
+    keys, complete = r.scan_iter(_META_PREFIX + "*", 2000)
+    if not keys:
+        why = ("task_ignore_result is on" if app.conf.task_ignore_result
+               else f"no results are stored: results may have expired (result_expires="
+                    f"{app.conf.result_expires}) or tasks use ignore_result=True")
+        return {"visible": False, "reason": f"Task failures are not visible because {why}. "
+                                             "Check logs for task errors instead."}
+    failures = []
+    for i in range(0, len(keys), 200):
+        for raw in r.cmd("MGET", *keys[i:i + 200]):
+            try:
+                meta = json.loads(raw) if raw else None
+            except ValueError:
+                continue  # pickle or other serializer
+            if meta and meta.get("status") == "FAILURE":
+                failures.append(meta)
+    failures.sort(key=lambda m: m.get("date_done") or "", reverse=True)
+    events, out = [], []
+    for m in failures[: max(1, min(int(limit), 100))]:
+        res = m.get("result") or {}
+        tb_tail = (m.get("traceback") or "").strip().splitlines()[-3:]
+        item = {"task_id": m.get("task_id"), "name": m.get("name"),
+                "date_done": m.get("date_done"),
+                "exception": preview(f"{res.get('exc_type')}: {res.get('exc_message')}", 300)
+                if isinstance(res, dict) else preview(res, 300),
+                "traceback_tail": [preview(l, 200) for l in tb_tail]}
+        out.append(item)
+        events.append(event(_parse_iso(m.get("date_done")), SOURCE, "task_failed",
+                            f"{item['name'] or 'task'}[{item['task_id']}] failed: {item['exception']}",
+                            "warning", task_id=item["task_id"]))
+    return {"visible": True, "results_scanned": len(keys), "scan_complete": complete,
+            "failed_count": len(failures), "failed": out, "events": events}
+
+
+def _parse_iso(value):
+    try:
+        return dt.datetime.fromisoformat(value) if value else None
+    except ValueError:
+        return None
+
+
+def task_details(task_id: str) -> dict:
+    if (s := _not_configured()):
+        return s
+    if not _TASK_ID_RE.match(task_id or ""):
+        return {"error": "Invalid task id"}
+    app = get_app()
+    out: dict = {"task_id": task_id}
+    url, reason = _backend(app)
+    if url:
+        raw = redis_client(url).cmd("GET", _META_PREFIX + task_id)
+        if raw:
+            try:
+                meta = json.loads(raw)
+                out["result_backend"] = {
+                    "status": meta.get("status"), "name": meta.get("name"),
+                    "date_done": meta.get("date_done"), "retries": meta.get("retries"),
+                    "worker": meta.get("worker"),
+                    "args": preview(meta.get("args"), 200), "kwargs": preview(meta.get("kwargs"), 200),
+                    "result": preview(meta.get("result"), 300),
+                    "traceback_tail": [preview(l, 200) for l in
+                                       (meta.get("traceback") or "").strip().splitlines()[-5:]],
+                }
+            except ValueError:
+                out["result_backend"] = {"note": "result is not JSON-encoded"}
+        else:
+            out["result_backend"] = {"note": "no stored result (pending, expired, or ignore_result)"}
+    else:
+        out["result_backend"] = {"note": f"not readable: {reason}"}
+    # Ask workers whether they currently hold the task (active/reserved/scheduled).
+    found = _inspect(app).query_task(task_id) or {}
+    out["on_workers"] = {w: {tid: {"state": state, "name": info.get("name"),
+                                   "args": preview(info.get("args"), 120)}
+                             for tid, (state, info) in tasks.items()}
+                         for w, tasks in found.items() if tasks}
+    return out

stackdoctor/checks/logs.py

@@ -0,0 +1,235 @@
+"""Log checks for files and docker containers listed in LOG_SOURCES.
+
+Only configured sources can be read, so the AI can't use this to read arbitrary files.
+"""
+
+from __future__ import annotations
+
+import datetime as dt
+import os
+import re
+import shutil
+import subprocess
+from collections import defaultdict
+
+from ..config import get_config, skipped
+from . import event, to_utc, utcnow
+
+SOURCE = "logs"
+MAX_LINE = 2000
+MAX_FILE_BYTES = 5_000_000
+
+_DOCKER_TS = re.compile(r"^(\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d)(?:\.(\d+))?Z\s?")
+_GENERIC_TS = re.compile(
+    r"(\d{4}-\d\d-\d\d)[T ](\d\d:\d\d:\d\d)(?:[.,](\d+))?\s*(Z|UTC|[+-]\d\d:?\d\d)?")
+
+# (kind, pattern, severity) — first match wins, so specific rules come first.
+RULES = [
+    ("worker_lost", r"WorkerLostError|worker lost|exited with 'signal 9|exited prematurely", "critical"),
+    ("worker_shutdown", r"\b(?:warm|cold) shutdown\b", "warning"),  # Celery: "worker: Warm shutdown (MainProcess)"
+    ("task_timeout", r"SoftTimeLimitExceeded|TimeLimitExceeded|(?:hard|soft) time limit", "warning"),
+    ("task_failed", r"Task [\w.]+\[[\w-]+\] raised unexpected|Task [\w.]+\[[\w-]+\] failed", "warning"),
+    ("db_lock_wait", r"still waiting for \w*Lock", "warning"),
+    ("db_lock_error", r"lock timeout|deadlock detected|could not obtain lock|LockNotAvailable", "warning"),
+    ("db_timeout", r"statement timeout|QueryCanceled|canceling statement", "warning"),
+    ("db_connection_error", r"could not connect to server|connection to server .* failed|too many clients|"
+                            r"remaining connection slots|server closed the connection", "warning"),
+    ("redis_error", r"OOM command not allowed|MISCONF|Error \d+ connecting to|redis\.exceptions\.\w*Error", "warning"),
+    ("timeout", r"\b504\b|Gateway Time-?out|ReadTimeout|timed out", "warning"),
+    ("error", r"\b(?:ERROR|CRITICAL|FATAL|PANIC)\b|Traceback \(most recent call last\)", "warning"),
+]
+_RULES = [(k, re.compile(p, re.IGNORECASE), s) for k, p, s in RULES]
+PROBLEM_KINDS = {k for k, _, _ in RULES}
+
+
+# ---------------------------------------------------------------------------
+# Sources
+# ---------------------------------------------------------------------------
+
+def _parse_source(spec: str) -> tuple[str, str]:
+    """'docker:api' / 'file:/x.log' / '/var/log/x.log' / 'api' -> (kind, target)."""
+    if spec.startswith("docker:"):
+        return "docker", spec[7:]
+    if spec.startswith("file:"):
+        return "file", spec[5:]
+    looks_like_path = any(c in spec for c in "/\\") or spec.endswith((".log", ".txt")) or os.path.exists(spec)
+    return ("file", os.path.expanduser(spec)) if looks_like_path else ("docker", spec)
+
+
+def _label(spec: str) -> str:
+    kind, target = _parse_source(spec)
+    return os.path.basename(target) if kind == "file" else f"docker:{target}"
+
+
+def _resolve(source: str) -> tuple[str, str, str] | None:
+    """Match a user-provided name against configured sources. Returns (label, kind, target)."""
+    for spec in get_config().log_sources:
+        kind, target = _parse_source(spec)
+        if source in (spec, target, os.path.basename(target)):
+            return spec, kind, target
+    return None
+
+
+def _parse_ts(line: str) -> tuple[dt.datetime | None, str]:
+    """Return (timestamp, line without docker's timestamp prefix)."""
+    m = _DOCKER_TS.match(line)
+    if m:
+        frac = (m.group(2) or "0")[:6].ljust(6, "0")
+        ts = dt.datetime.fromisoformat(f"{m.group(1)}.{frac}+00:00")
+        return ts, line[m.end():]
+    m = _GENERIC_TS.search(line[:80])
+    if m:
+        date, clock, frac, tz = m.groups()
+        frac = (frac or "0")[:6].ljust(6, "0")
+        tz = "+00:00" if tz in ("Z", "UTC") else (tz if tz and ":" in tz else (f"{tz[:3]}:{tz[3:]}" if tz else ""))
+        try:
+            return to_utc(dt.datetime.fromisoformat(f"{date}T{clock}.{frac}{tz}")), line
+        except ValueError:
+            pass
+    return None, line
+
+
+def _read_file(path: str, max_lines: int) -> list[str]:
+    with open(path, "rb") as f:
+        f.seek(0, os.SEEK_END)
+        start = max(0, f.tell() - MAX_FILE_BYTES)
+        f.seek(start)
+        data = f.read()
+    lines = data.decode("utf-8", errors="replace").splitlines()
+    if start > 0:
+        lines = lines[1:]  # first line is probably partial
+    return lines[-max_lines:]
+
+
+def _read_docker(container: str, max_lines: int, since_minutes: float | None) -> list[str]:
+    if not shutil.which("docker"):
+        raise RuntimeError("docker CLI not found on PATH")
+    cmd = ["docker", "logs", "--timestamps", "--tail", str(max_lines)]
+    if since_minutes:
+        cmd += ["--since", f"{int(since_minutes * 60)}s"]
+    proc = subprocess.run(cmd + [container], capture_output=True, timeout=6,
+                          text=True, encoding="utf-8", errors="replace")
+    if proc.returncode != 0:
+        raise RuntimeError(proc.stderr.strip()[:300] or f"docker logs exited {proc.returncode}")
+    # Container stdout and stderr arrive separately; timestamps let us re-interleave them.
+    return sorted(proc.stdout.splitlines() + proc.stderr.splitlines())
+
+
+def _read(kind: str, target: str, max_lines: int, since_minutes: float | None = None) -> list[dict]:
+    raw = _read_file(target, max_lines) if kind == "file" else _read_docker(target, max_lines, since_minutes)
+    out, last_ts, trailing = [], None, []
+    for line in raw:
+        ts, text = _parse_ts(line)
+        if ts:
+            last_ts, trailing = ts, []
+        entry = {"ts": ts or last_ts, "line": text[:MAX_LINE]}  # tracebacks inherit the last timestamp
+        if not ts:
+            trailing.append(entry)
+        out.append(entry)
+    if kind == "file" and trailing:
+        # Lines after the last timestamp were written by the file's last modification at the latest.
+        # This dates lines printed without a timestamp, like Celery's "worker: Warm shutdown (MainProcess)",
+        # which Celery writes straight to stdout (it never goes through logging or --logfile).
+        mtime = to_utc(os.path.getmtime(target))
+        for entry in trailing:
+            entry["ts"] = max(entry["ts"], mtime) if entry["ts"] else mtime
+    return out
+
+
+# ---------------------------------------------------------------------------
+# Tools
+# ---------------------------------------------------------------------------
+
+def tail_logs(source: str, lines: int = 100) -> dict:
+    cfg = get_config()
+    if not cfg.log_sources:
+        return skipped("LOG_SOURCES not set")
+    resolved = _resolve(source)
+    if not resolved:
+        return {"error": f"Unknown log source '{source}'.", "configured_sources": cfg.log_sources}
+    label, kind, target = resolved
+    entries = _read(kind, target, max(1, min(int(lines), 500)))
+    return {"source": label, "count": len(entries), "lines": entries}
+
+
+def search_logs(pattern: str, since_minutes: int = 30, source: str | None = None) -> dict:
+    cfg = get_config()
+    if not cfg.log_sources:
+        return skipped("LOG_SOURCES not set")
+    try:
+        regex = re.compile(pattern, re.IGNORECASE)
+    except re.error:
+        regex = re.compile(re.escape(pattern), re.IGNORECASE)
+    since = utcnow() - dt.timedelta(minutes=max(1, min(int(since_minutes), 24 * 60)))
+    targets = [_resolve(source)] if source else [(s, *_parse_source(s)) for s in cfg.log_sources]
+    if source and not targets[0]:
+        return {"error": f"Unknown log source '{source}'.", "configured_sources": cfg.log_sources}
+
+    matches, errors = [], {}
+    for label, kind, target in targets:
+        try:
+            entries = _read(kind, target, 20_000, since_minutes)
+        except Exception as e:
+            errors[label] = f"{type(e).__name__}: {e}"
+            continue
+        for e in entries:
+            if (e["ts"] is None or e["ts"] >= since) and regex.search(e["line"]):
+                matches.append({"source": label, **e})
+    matches.sort(key=lambda m: m["ts"] or since)
+    return {"pattern": pattern, "since": since, "match_count": len(matches),
+            "matches": matches[-200:], "errors": errors or None}
+
+
+def classify(line: str) -> tuple[str, str] | None:
+    for kind, rx, severity in _RULES:
+        if rx.search(line):
+            return kind, severity
+    return None
+
+
+def recent_problems(since_minutes: int = 30) -> dict:
+    """Classify recent log lines into problem kinds, aggregated per source and kind."""
+    cfg = get_config()
+    if not cfg.log_sources:
+        return skipped("LOG_SOURCES not set")
+    now = utcnow()
+    since = now - dt.timedelta(minutes=since_minutes)
+    spike_since = now - dt.timedelta(minutes=5)
+    groups: dict[tuple[str, str], dict] = {}
+    recent_errors: dict[str, list] = defaultdict(list)
+    errors, last_activity = {}, {}
+    for spec in cfg.log_sources:
+        kind, target = _parse_source(spec)
+        try:
+            entries = _read(kind, target, 20_000, since_minutes)
+        except Exception as e:
+            errors[spec] = f"{type(e).__name__}: {e}"
+            continue
+        last_activity[_label(spec)] = max((e["ts"] for e in entries if e["ts"]), default=None)
+        for e in entries:
+            if e["ts"] and e["ts"] < since:
+                continue
+            hit = classify(e["line"])
+            if not hit:
+                continue
+            k, sev = hit
+            g = groups.setdefault((spec, k), {"source": spec, "kind": k, "severity": sev, "count": 0,
+                                              "first": e["ts"], "last": e["ts"], "sample": e["line"][:300]})
+            g["count"] += 1
+            g["last"] = e["ts"] or g["last"]
+            if e["ts"] and e["ts"] >= spike_since:
+                recent_errors[spec].append(e["ts"])
+
+    events = []
+    for g in groups.values():
+        events.append(event(g["first"], SOURCE, g["kind"],
+                            f"[{_label(g['source'])}] {g['count']}x {g['kind']} (last {g['last'] and g['last'].isoformat(timespec='seconds')}): {g['sample']}",
+                            g["severity"], log_source=g["source"], count=g["count"], last=g["last"]))
+    for spec, stamps in recent_errors.items():
+        if len(stamps) >= cfg.log_error_spike:
+            events.append(event(min(stamps), SOURCE, "error_spike",
+                                f"[{_label(spec)}] {len(stamps)} problem lines in the last 5 minutes "
+                                f"(threshold {cfg.log_error_spike})", "warning", log_source=spec))
+    summary = sorted(groups.values(), key=lambda g: (g["first"] or now))
+    return {"since_minutes": since_minutes, "problems": summary, "last_activity": last_activity,
+            "errors": errors or None, "events": events}