stackdoctor 0.1.0__py3-none-any.whl

stackdoctor/diagnose.py

@@ -0,0 +1,248 @@
+"""diagnose(): run checks concurrently, merge one timeline, derive findings and possible causes."""
+
+from __future__ import annotations
+
+import asyncio
+import datetime as dt
+import time
+
+from .checks import celery, event, logs, postgres, redis, utcnow
+from .config import get_config
+
+# name -> (function, kind of timeout)
+CHECKS = {
+    "postgres.active_queries": (postgres.active_queries, "default"),
+    "postgres.blocking_locks": (postgres.blocking_locks, "default"),
+    "postgres.slow_queries": (postgres.slow_queries, "default"),
+    "celery.workers": (celery.workers, "celery"),
+    "celery.queue_lengths": (celery.queue_lengths, "celery"),
+    "celery.failed_tasks": (celery.failed_tasks, "default"),
+    "redis.memory_and_clients": (redis.memory_and_clients, "default"),
+    "logs.recent_problems": (logs.recent_problems, "default"),
+}
+
+_JOBS = {"celery.workers", "celery.queue_lengths", "celery.failed_tasks", "postgres.blocking_locks",
+         "postgres.active_queries", "redis.memory_and_clients", "logs.recent_problems"}
+_SLOW = {"postgres.active_queries", "postgres.blocking_locks", "postgres.slow_queries",
+         "redis.memory_and_clients", "celery.workers", "celery.queue_lengths", "logs.recent_problems"}
+_REDIS = {"redis.memory_and_clients", "celery.queue_lengths", "logs.recent_problems"}
+ROUTES = [
+    (("job", "task", "queue", "worker", "celery", "stuck", "backlog", "pending"), _JOBS),
+    (("slow", "latency", "api", "timeout", "query", "database", "db", "lock", "postgres", "hang"), _SLOW),
+    (("redis", "memory", "cache", "oom", "evict", "broker"), _REDIS),
+]
+
+# Conditions observed "now" that are still in effect, so later effects can follow them.
+ONGOING = {"lock_held", "idle_in_transaction", "query_blocked", "long_query", "no_workers",
+           "worker_missing", "workers_saturated", "task_long_running", "queue_backlog",
+           "queue_no_consumer", "memory_high"}
+
+CHAIN_RULES = [
+    {
+        "hypothesis": "Celery worker went down → queue is not being consumed",
+        "causes": {"worker_shutdown", "worker_lost", "no_workers", "worker_missing"},
+        "effects": {"queue_backlog", "queue_no_consumer", "task_timeout", "timeout"},
+        "explanation": "With no (or fewer) workers consuming, messages accumulate in the broker "
+                       "and callers waiting on results time out.",
+    },
+    {
+        "hypothesis": "Database lock → blocked queries → stuck or failing tasks",
+        "causes": {"lock_held", "idle_in_transaction"},
+        "effects": {"query_blocked", "db_lock_wait", "task_long_running", "workers_saturated", "queue_backlog",
+                    "task_timeout", "db_lock_error", "db_timeout", "task_failed", "timeout"},
+        "explanation": "A transaction holding a lock makes other queries wait; tasks running those "
+                       "queries hang, occupy worker slots, and eventually time out or fail.",
+    },
+    {
+        "hypothesis": "Long-running query → slow tasks/requests",
+        "causes": {"long_query"},
+        "effects": {"task_long_running", "workers_saturated", "db_timeout", "task_timeout", "timeout", "queue_backlog"},
+        "explanation": "A slow query (e.g. a sequential scan) occupies the database and the "
+                       "code paths waiting on it.",
+    },
+    {
+        "hypothesis": "Redis memory pressure → broker/cache errors",
+        "causes": {"memory_high", "evictions"},
+        "effects": {"redis_error", "task_failed", "error_spike", "queue_backlog"},
+        "explanation": "Near maxmemory Redis evicts keys or rejects writes (OOM), which breaks "
+                       "the broker, result backend or cache.",
+    },
+    {
+        "hypothesis": "Database connection problems → task failures",
+        "causes": {"db_connection_error"},
+        "effects": {"task_failed", "error_spike", "timeout"},
+        "explanation": "Workers or the API can't reach Postgres (down, or out of connections).",
+    },
+    {
+        "hypothesis": "All worker slots busy → queue backlog",
+        "causes": {"workers_saturated", "task_long_running"},
+        "effects": {"queue_backlog"},
+        "explanation": "Tasks are being consumed slower than they arrive.",
+    },
+]
+
+_SEVERITY = {"critical": 0, "warning": 1, "info": 2}
+
+
+def select_checks(symptom: str) -> list[str]:
+    text = (symptom or "").lower()
+    chosen: set[str] = set()
+    for words, checks in ROUTES:
+        if any(w in text for w in words):
+            chosen |= checks
+    return sorted(chosen or CHECKS)  # vague symptom: run everything
+
+
+async def _run(name: str) -> dict:
+    cfg = get_config()
+    fn, kind = CHECKS[name]
+    timeout = cfg.check_timeout_s
+    if kind == "celery":  # ping + 4 inspect broadcasts, each waiting up to the inspect timeout
+        timeout = max(timeout, cfg.celery_inspect_timeout_s * 6 + 2)
+    t0 = time.perf_counter()
+    try:
+        result = await asyncio.wait_for(asyncio.to_thread(fn), timeout)
+        status = "skipped" if "skipped" in result else "ok"
+        out = {"status": status, "result": result}
+    except TimeoutError:
+        out = {"status": "timeout", "error": f"timed out after {timeout:.0f}s"}
+    except Exception as e:
+        out = {"status": "error", "error": f"{type(e).__name__}: {str(e)[:300]}"}
+    out["ms"] = round((time.perf_counter() - t0) * 1000)
+    return out
+
+
+def _cross_check_queues(results: dict) -> list[dict]:
+    """Queues with messages that no live worker consumes (Redis brokers don't report consumers)."""
+    w = results.get("celery.workers", {}).get("result") or {}
+    q = results.get("celery.queue_lengths", {}).get("result") or {}
+    if "alive_count" not in w or "queues" not in q:
+        return []
+    consumed = {name for worker in w["workers"] for name in worker["queues"]}
+    events = []
+    for name, info in q["queues"].items():
+        n = info.get("messages") or 0
+        if n and name not in consumed and info.get("consumers") is None:
+            events.append(event(None, "celery", "queue_no_consumer",
+                                f"Queue '{name}' has {n} messages and no live worker consumes it",
+                                "critical", queue=name))
+    return events
+
+
+def _fmt(e: dict) -> str:
+    return f"{e['ts'].isoformat(timespec='seconds')} {e['source']}.{e['kind']}: {e['detail']}"
+
+
+def find_chains(events: list[dict], window_min: float) -> list[dict]:
+    window = dt.timedelta(minutes=window_min)
+    slack = dt.timedelta(seconds=60)
+    chains, used = [], []
+    for rule in CHAIN_RULES:
+        causes = sorted((e for e in events if e["kind"] in rule["causes"]), key=lambda e: e["ts"])
+        if not causes:
+            continue
+        cause = causes[0]
+        ongoing = cause["kind"] in ONGOING
+        effects: dict[str, dict] = {}
+        for e in sorted(events, key=lambda e: e["ts"]):
+            if e is cause or e["kind"] not in rule["effects"] or e["kind"] in effects:
+                continue
+            after = e["ts"] >= cause["ts"] - slack
+            close = ongoing or e["ts"] <= cause["ts"] + window
+            if after and close:
+                effects[e["kind"]] = e
+        if not effects:
+            continue
+        ids = {id(cause), *map(id, effects.values())}
+        if any(ids <= prev for prev in used):
+            continue  # already explained by an earlier chain
+        used.append(ids)
+        sources = {e["source"] for e in effects.values()} | {cause["source"]}
+        confidence = "medium" if len(effects) >= 2 and len(sources) >= 2 else "low"
+        chains.append({
+            "label": "possible cause",
+            "hypothesis": rule["hypothesis"],
+            "confidence": confidence,
+            "cause": _fmt(cause),
+            "effects": [_fmt(e) for e in effects.values()],
+            "evidence_window": {"from": cause["ts"], "to": max(e["ts"] for e in effects.values())},
+            "explanation": rule["explanation"],
+            "caveat": "Inferred from timing only. Verify before acting; this is not a confirmed root cause.",
+        })
+    chains.sort(key=lambda c: c["confidence"] != "medium")
+    return chains
+
+
+def _findings(events: list[dict], results: dict) -> tuple[list[dict], list[str]]:
+    grouped: dict[tuple, dict] = {}
+    for e in events:
+        if e["severity"] == "info":
+            continue
+        key = (e["source"], e["kind"])
+        f = grouped.setdefault(key, {"severity": e["severity"], "source": e["source"], "kind": e["kind"],
+                                     "count": 0, "first_seen": e["ts"], "example": e["detail"]})
+        f["count"] += e.get("count", 1)
+        if _SEVERITY[e["severity"]] < _SEVERITY[f["severity"]]:
+            f["severity"] = e["severity"]
+    findings = sorted(grouped.values(), key=lambda f: (_SEVERITY[f["severity"]], f["first_seen"]))
+
+    notes = []
+    ft = results.get("celery.failed_tasks", {}).get("result") or {}
+    if ft.get("visible") is False:
+        n = sum(1 for e in events if e["kind"] == "task_failed" and e["source"] == "logs")
+        notes.append(f"{ft['reason']} Falling back to logs: {n} task-failure group(s) found in logs.")
+    sq = results.get("postgres.slow_queries", {}).get("result") or {}
+    if sq.get("available") is False:
+        notes.append("pg_stat_statements is not installed; slow query stats unavailable (see postgres.slow_queries).")
+    lg = results.get("logs.recent_problems", {}).get("result") or {}
+    kinds = {e["kind"] for e in events}
+    if kinds & {"no_workers", "worker_missing"} and not kinds & {"worker_shutdown", "worker_lost"}:
+        seen = ", ".join(f"{k} last written {v.isoformat(timespec='seconds')}"
+                         for k, v in (lg.get("last_activity") or {}).items() if v)
+        notes.append("Workers are missing but no shutdown/crash line was found in the logs. Celery prints "
+                     "'worker: Warm shutdown' to stdout only (not to --logfile), so LOG_SOURCES must include "
+                     "the worker's stdout (docker logs, or a file stdout is redirected to)."
+                     + (f" Log activity: {seen}." if seen else ""))
+    if lg.get("errors"):
+        notes.append(f"Some log sources could not be read: {lg['errors']}")
+    return findings, notes
+
+
+async def diagnose(symptom: str = "") -> dict:
+    cfg = get_config()
+    started = utcnow()
+    names = select_checks(symptom)
+    outcomes = await asyncio.gather(*(_run(n) for n in names))
+    results = dict(zip(names, outcomes))
+
+    events: list[dict] = []
+    for out in results.values():
+        result = out.get("result")
+        if isinstance(result, dict):
+            events.extend(result.pop("events", []))
+    events.extend(_cross_check_queues(results))
+    # A lock holder is also a "long query"; the lock is the more specific explanation.
+    lock_pids = {e.get("pid") for e in events if e["kind"] == "lock_held"}
+    events = [e for e in events if not (e["kind"] == "long_query" and e.get("pid") in lock_pids)]
+    for e in events:
+        if e["ts"] > started + dt.timedelta(seconds=30):  # clock skew guard
+            e["ts"] = started
+
+    findings, notes = _findings(events, results)
+    chains = find_chains(events, cfg.chain_window_min)
+    timeline = [{"ts": e["ts"], "source": e["source"], "kind": e["kind"],
+                 "severity": e["severity"], "detail": e["detail"]}
+                for e in sorted(events, key=lambda e: e["ts"])][-80:]
+
+    return {
+        "timestamp": started,
+        "symptom": symptom,
+        "checks_run": {n: r["status"] for n, r in results.items()},
+        "findings": findings,
+        "possible_causes": chains,
+        "notes": notes,
+        "timeline": timeline,
+        "checks": {n: r for n, r in results.items() if r["status"] != "skipped"},
+        "skipped": {n: r["result"]["skipped"] for n, r in results.items() if r["status"] == "skipped"},
+        "duration_ms": round((utcnow() - started).total_seconds() * 1000),
+    }

stackdoctor/safety.py

@@ -0,0 +1,331 @@
+"""Safety layer: SQL validation, Redis command allowlist, secret redaction, output caps.
+
+Everything that leaves stackdoctor goes through `safe_output()`.
+"""
+
+from __future__ import annotations
+
+import datetime as dt
+import decimal
+import json
+import re
+import uuid
+from typing import Any
+
+import sqlglot
+from sqlglot import exp
+from sqlglot.errors import ParseError, TokenError
+from sqlglot.tokens import TokenType
+
+
+class UnsafeError(ValueError):
+    """Raised when a request would break the read-only guarantee."""
+
+
+# ---------------------------------------------------------------------------
+# SQL validation
+# ---------------------------------------------------------------------------
+
+# Functions that have side effects or read server files, even inside a
+# read-only transaction (or that execute arbitrary SQL from a string).
+FORBIDDEN_FUNCTIONS = {
+    "pg_terminate_backend", "pg_cancel_backend", "pg_reload_conf",
+    "pg_read_file", "pg_read_binary_file", "pg_ls_dir", "pg_stat_file",
+    "set_config", "pg_rotate_logfile", "pg_switch_wal", "pg_promote",
+    "pg_create_restore_point", "pg_log_backend_memory_contexts",
+    "pg_notify", "pg_sleep", "pg_sleep_for", "pg_sleep_until",
+    "nextval", "setval",
+    "query_to_xml", "query_to_xml_and_xmlschema", "query_to_xmlschema",
+    "cursor_to_xml", "cursor_to_xmlschema",
+    "pg_file_write", "pg_file_rename", "pg_file_unlink", "pg_logdir_ls",
+}
+FORBIDDEN_PREFIXES = (
+    "lo_", "dblink", "pg_advisory_", "pg_try_advisory_", "pg_ls_",
+    "pg_read_", "pg_replication_", "pg_create_", "pg_drop_",
+)
+
+# Nodes that must never appear anywhere in the tree (catches writable CTEs).
+FORBIDDEN_NODES = (
+    exp.Insert, exp.Update, exp.Delete, exp.Merge, exp.Create, exp.Drop,
+    exp.Alter, exp.Command, exp.Into, exp.Lock, exp.Copy, exp.Set,
+    exp.Transaction, exp.Commit, exp.Rollback, exp.TruncateTable,
+)
+
+_EXPLAIN_SAFE_OPTIONS = {"VERBOSE", "COSTS", "SETTINGS", "BUFFERS", "FORMAT",
+                         "SUMMARY", "GENERIC_PLAN", "TEXT", "JSON", "YAML",
+                         "XML", "TRUE", "FALSE", "ON", "OFF", "1", "0"}
+
+
+def _function_name(node: exp.Func) -> str:
+    if isinstance(node, exp.Anonymous):
+        return str(node.name).lower()
+    return node.sql_name().lower()
+
+
+def _check_select_tree(tree: exp.Expression) -> None:
+    if not isinstance(tree, (exp.Select, exp.Union, exp.Intersect, exp.Except)):
+        raise UnsafeError(f"Only SELECT / WITH / EXPLAIN are allowed (got {tree.key.upper()}).")
+    for node in tree.walk():
+        if isinstance(node, FORBIDDEN_NODES):
+            raise UnsafeError(f"Statement contains a forbidden clause: {node.key.upper()}.")
+        if isinstance(node, exp.Select) and node.args.get("into"):
+            raise UnsafeError("SELECT ... INTO is not allowed.")
+        if isinstance(node, exp.Func):
+            name = _function_name(node)
+            if name in FORBIDDEN_FUNCTIONS or name.startswith(FORBIDDEN_PREFIXES):
+                raise UnsafeError(f"Function {name}() is not allowed (side effects or file access).")
+
+
+def _parse_single(sql: str) -> exp.Expression:
+    try:
+        statements = [s for s in sqlglot.parse(sql, dialect="postgres")
+                      if s is not None and not isinstance(s, exp.Semicolon)]
+    except (ParseError, TokenError) as e:
+        raise UnsafeError(f"Could not parse SQL: {str(e).splitlines()[0]}") from None
+    if len(statements) != 1:
+        raise UnsafeError("Exactly one statement is allowed.")
+    return statements[0]
+
+
+def _strip_trailing(sql: str) -> str:
+    """Return the SQL without trailing semicolons/comments, using the tokenizer."""
+    tokens = [t for t in sqlglot.tokenize(sql, dialect="postgres") if t.token_type != TokenType.SEMICOLON]
+    if not tokens:
+        raise UnsafeError("Empty SQL.")
+    return sql[: tokens[-1].end + 1]
+
+
+def validate_select(sql: str, limit: int) -> str:
+    """Validate `sql` as a single read-only query and return the SQL to execute.
+
+    SELECT / WITH queries are wrapped so a row LIMIT always applies.
+    EXPLAIN (without ANALYZE) is returned unwrapped.
+    """
+    if not sql or not sql.strip():
+        raise UnsafeError("Empty SQL.")
+    try:
+        tokens = sqlglot.tokenize(sql, dialect="postgres")
+    except TokenError as e:
+        raise UnsafeError(f"Could not tokenize SQL: {e}") from None
+    if not tokens:
+        raise UnsafeError("Empty SQL.")
+
+    if tokens[0].text.upper() == "EXPLAIN":
+        # The postgres tokenizer swallows everything after EXPLAIN as one string,
+        # so tokenize the remainder on its own.
+        sql = sql[tokens[0].end + 1:]
+        tokens = sqlglot.tokenize(sql, dialect="postgres")
+        i = 0
+        if i < len(tokens) and tokens[i].token_type == TokenType.L_PAREN:
+            depth = 0
+            while i < len(tokens):
+                t = tokens[i]
+                if t.token_type == TokenType.L_PAREN:
+                    depth += 1
+                elif t.token_type == TokenType.R_PAREN:
+                    depth -= 1
+                    if depth == 0:
+                        i += 1
+                        break
+                elif t.text.upper() not in _EXPLAIN_SAFE_OPTIONS and t.token_type != TokenType.COMMA:
+                    raise UnsafeError(f"EXPLAIN option {t.text.upper()} is not allowed (no ANALYZE).")
+                i += 1
+        else:
+            while i < len(tokens) and tokens[i].text.upper() in {"VERBOSE", "ANALYZE", "ANALYSE"}:
+                if tokens[i].text.upper() in {"ANALYZE", "ANALYSE"}:
+                    raise UnsafeError("EXPLAIN ANALYZE executes the query and is not allowed.")
+                i += 1
+        if i >= len(tokens):
+            raise UnsafeError("EXPLAIN needs a query.")
+        inner = sql[tokens[i].start:]
+        _check_select_tree(_parse_single(inner))
+        return "EXPLAIN " + _strip_trailing(sql)
+
+    _check_select_tree(_parse_single(sql))
+    body = _strip_trailing(sql)
+    # Newlines keep a trailing `-- comment` from swallowing the wrapper.
+    return f"SELECT * FROM (\n{body}\n) AS sd_sub LIMIT {int(limit)}"
+
+
+# ---------------------------------------------------------------------------
+# Redis allowlist (subcommand-aware)
+# ---------------------------------------------------------------------------
+
+# None = no subcommand check; a set = only these subcommands are allowed.
+REDIS_ALLOWED: dict[str, set[str] | None] = {
+    "PING": None, "INFO": None, "DBSIZE": None, "SCAN": None, "TYPE": None,
+    "TTL": None, "PTTL": None, "EXISTS": None, "GET": None, "MGET": None,
+    "GETRANGE": None, "STRLEN": None, "LLEN": None, "LRANGE": None,
+    "HLEN": None, "HSCAN": None, "SCARD": None, "SSCAN": None,
+    "ZCARD": None, "ZRANGE": None, "XLEN": None,
+    "CLIENT": {"LIST", "INFO"},
+    "OBJECT": {"ENCODING", "IDLETIME", "FREQ"},
+    "MEMORY": {"USAGE", "STATS"},
+    "CONFIG": {"GET"},
+    "SLOWLOG": {"GET", "LEN"},
+}
+
+
+def check_redis_command(*args: Any) -> None:
+    """Raise UnsafeError unless the command is on the read-only allowlist.
+
+    Accepts both ("CLIENT", "LIST") and ("CLIENT LIST",) forms, like redis-py.
+    """
+    if not args:
+        raise UnsafeError("Empty Redis command.")
+    parts = str(args[0]).split() + [str(a) for a in args[1:]]
+    if not parts:
+        raise UnsafeError("Empty Redis command.")
+    cmd = parts[0].upper()
+    if cmd not in REDIS_ALLOWED:
+        raise UnsafeError(f"Redis command {cmd} is not on the read-only allowlist.")
+    subs = REDIS_ALLOWED[cmd]
+    if subs is not None:
+        sub = parts[1].upper() if len(parts) > 1 else ""
+        if sub not in subs:
+            raise UnsafeError(f"Redis command {cmd} {sub} is not on the read-only allowlist.")
+
+
+# ---------------------------------------------------------------------------
+# Redaction
+# ---------------------------------------------------------------------------
+
+MASK = "***"
+_SECRET_WORDS = r"(?:password|passwd|pwd|secret|token|api[_-]?key|apikey|access[_-]?key|private[_-]?key|auth|credential|session[_-]?id|cookie|dsn)"
+SENSITIVE_KEY_RE = re.compile(rf"(?i)^[\w.-]*{_SECRET_WORDS}[\w.-]*$")
+
+_PATTERNS: list[tuple[re.Pattern, str]] = [
+    # scheme://user:password@host  ->  scheme://user:***@host
+    (re.compile(r"(?i)\b([a-z][a-z0-9+.-]*://[^:/\s@]*:)([^@\s/]+)@"), rf"\1{MASK}@"),
+    # Authorization / Proxy-Authorization headers
+    (re.compile(r"(?i)\b((?:proxy-)?authorization[\"']?\s*[:=]\s*[\"']?)(?:(bearer|basic|token|digest)\s+)?[^\s\"',}]+"),
+     lambda m: f"{m.group(1)}{(m.group(2) + ' ') if m.group(2) else ''}{MASK}"),
+    # Bare bearer tokens
+    (re.compile(r"(?i)\b(bearer\s+)[a-z0-9._~+/=-]{8,}"), rf"\1{MASK}"),
+    # KEY=value, key: value, "key": "value" where key looks secret
+    (re.compile(rf"(?i)([\"']?\b[\w.-]*{_SECRET_WORDS}[\w.-]*[\"']?\s*[:=]\s*)(\"[^\"]*\"|'[^']*'|[^\s,;&}}]+)"),
+     lambda m: m.group(1) + (m.group(2)[0] + MASK + m.group(2)[0] if m.group(2)[:1] in "\"'" else MASK)),
+    # Well-known token formats
+    (re.compile(r"\b(?:AKIA|ASIA)[0-9A-Z]{16}\b"), MASK),                     # AWS access key id
+    (re.compile(r"\bgh[pousr]_[A-Za-z0-9]{20,}\b"), MASK),                    # GitHub
+    (re.compile(r"\bxox[abprs]-[A-Za-z0-9-]{10,}\b"), MASK),                  # Slack
+    (re.compile(r"\b(?:sk|pk|rk)_(?:live|test)_[A-Za-z0-9]{10,}\b"), MASK),   # Stripe
+    (re.compile(r"\bsk-[A-Za-z0-9_-]{20,}\b"), MASK),                         # OpenAI / Anthropic style
+    (re.compile(r"\bAIza[0-9A-Za-z_-]{30,}\b"), MASK),                        # Google API key
+    (re.compile(r"\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b"), MASK),  # JWT
+    (re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?(?:-----END [A-Z ]*PRIVATE KEY-----|$)"), MASK),
+]
+
+
+def redact_text(text: str) -> str:
+    for pattern, repl in _PATTERNS:
+        text = pattern.sub(repl, text)
+    return text
+
+
+def redact(obj: Any) -> Any:
+    """Recursively redact secrets in strings and in values under secret-looking keys."""
+    if isinstance(obj, str):
+        return redact_text(obj)
+    if isinstance(obj, dict):
+        out = {}
+        for k, v in obj.items():
+            if isinstance(k, str) and SENSITIVE_KEY_RE.match(k) and isinstance(v, (str, int, float, bytes)):
+                out[k] = MASK
+            else:
+                out[k] = redact(v)
+        return out
+    if isinstance(obj, (list, tuple)):
+        return [redact(v) for v in obj]
+    return obj
+
+
+def preview(value: Any, max_chars: int = 120) -> str:
+    """Short, redacted, single-line preview of any value (task args, Redis values...)."""
+    text = value if isinstance(value, str) else repr(value)
+    text = redact_text(" ".join(text.split()))
+    return text if len(text) <= max_chars else text[: max_chars - 1] + "…"
+
+
+# ---------------------------------------------------------------------------
+# JSON + output caps
+# ---------------------------------------------------------------------------
+
+def _json_default(o: Any) -> Any:
+    if isinstance(o, dt.datetime):
+        if o.tzinfo is None:
+            o = o.astimezone()
+        return o.astimezone(dt.timezone.utc).isoformat(timespec="seconds")
+    if isinstance(o, (dt.date, dt.time)):
+        return o.isoformat()
+    if isinstance(o, dt.timedelta):
+        return round(o.total_seconds(), 1)
+    if isinstance(o, decimal.Decimal):
+        return float(o)
+    if isinstance(o, (uuid.UUID, memoryview)):
+        return str(o)
+    if isinstance(o, bytes):
+        return o.decode("utf-8", errors="replace")
+    if isinstance(o, (set, frozenset)):
+        return sorted(o, key=str)
+    return str(o)
+
+
+def to_jsonable(obj: Any) -> Any:
+    return json.loads(json.dumps(obj, default=_json_default))
+
+
+def _largest_list(obj: Any, best: tuple[int, list | None] = (0, None)) -> tuple[int, list | None]:
+    if isinstance(obj, list):
+        if len(obj) > best[0]:
+            best = (len(obj), obj)
+        for v in obj:
+            best = _largest_list(v, best)
+    elif isinstance(obj, dict):
+        for v in obj.values():
+            best = _largest_list(v, best)
+    return best
+
+
+def _shorten_strings(obj: Any, max_len: int) -> Any:
+    if isinstance(obj, str):
+        return obj if len(obj) <= max_len else obj[:max_len] + "…"
+    if isinstance(obj, dict):
+        return {k: _shorten_strings(v, max_len) for k, v in obj.items()}
+    if isinstance(obj, list):
+        return [_shorten_strings(v, max_len) for v in obj]
+    return obj
+
+
+def cap_output(obj: Any, max_chars: int) -> Any:
+    """Shrink a JSON-able object until it serializes under `max_chars`."""
+    size = lambda o: len(json.dumps(o))  # noqa: E731
+    if size(obj) <= max_chars:
+        return obj
+    truncated = False
+    for _ in range(40):
+        n, lst = _largest_list(obj)
+        if lst is None or n <= 3:
+            break
+        dropped = n - n // 2
+        del lst[n // 2:]
+        lst.append({"_truncated": f"{dropped} more items omitted"})
+        truncated = True
+        if size(obj) <= max_chars:
+            break
+    for max_len in (1000, 300, 100):
+        if size(obj) <= max_chars:
+            break
+        obj = _shorten_strings(obj, max_len)
+        truncated = True
+    if size(obj) > max_chars:
+        text = json.dumps(obj)
+        return {"_truncated": True, "preview": text[: max_chars - 100]}
+    if truncated and isinstance(obj, dict):
+        obj["_output_truncated"] = True
+    return obj
+
+
+def safe_output(obj: Any, max_chars: int) -> Any:
+    """Final gate for every tool response: JSON-safe, redacted, size-capped."""
+    return cap_output(redact(to_jsonable(obj)), max_chars)