codebase-explorer 2.2.0__py3-none-any.whl

cbe/__init__.py

@@ -0,0 +1,9 @@
+"""Codebase Explorer semantic documentation product."""
+
+from importlib.metadata import PackageNotFoundError, version
+
+try:
+    __version__ = version("codebase-explorer")
+except PackageNotFoundError:
+    # Source checkout without installed distribution metadata.
+    __version__ = "source"

cbe/__main__.py

@@ -0,0 +1,4 @@
+from cbe.cli import main
+
+if __name__ == "__main__":
+    raise SystemExit(main())

cbe/aggregation.py

@@ -0,0 +1,183 @@
+"""Unit records and edges for the tree, and the relationship view of a finished tree.
+
+The tree itself comes from ``skeleton`` (structure) and ``treename`` (names).
+"""
+from __future__ import annotations
+
+from collections import Counter, defaultdict
+
+from cbe.records import build_records, frozen_sources
+from cbe.docs import unit_ranges
+
+
+def inputs(units: dict, dispositions: dict, inventory: dict, context: dict) -> tuple[dict, list[dict]]:
+    """Use the same source-coordinate ownership as the published catalog."""
+    ranges = {uid: unit_ranges(uid, unit, inventory) for uid, unit in units.items()}
+    edge_table = {}
+    records, _ = build_records(
+        units, dispositions, inventory, ranges, frozen_sources(context, list(inventory)),
+        {f["path"]: f.get("language", "unknown") for f in context.get("files", [])},
+        context, context.get("graph"), edge_table)
+    return records, unit_edges(records, edge_table)
+
+
+def unit_edges(records: dict, edge_table: dict) -> list[dict]:
+    """Roll resolved raw edges through the catalog's folded-symbol ownership."""
+    owners = {sid: uid for uid, row in records.items()
+              for sid in [uid] + row.get("folded_members", [])}
+    counts = Counter()
+    for edge in edge_table.values():
+        source, target = owners.get(edge.get("source_symbol")), owners.get(edge.get("target_symbol"))
+        if edge.get("status") == "resolved" and source and target and source != target:
+            counts[source, target, edge["kind"]] += 1
+    # File-level imports without two source-coordinate unit owners are not
+    # unit edges: they are never attributed to an arbitrary function in the imported file.
+    for uid, unit in records.items():
+        for relation in unit.get("relations") or []:
+            target = owners.get(relation.get("target"))
+            if target and target != uid:
+                counts[uid, target, "relation"] += 1
+    return [{"from": a, "to": b, "kind": kind, "count": n,
+                      **({"provenance": "model_inferred"} if kind == "relation" else {})}
+                     for (a, b, kind), n in sorted(counts.items())]
+
+
+def roll_up_edges(edges: list[dict], membership: dict[str, str]) -> list[dict]:
+    counts = Counter()
+    for e in edges:
+        a, b = membership.get(e["from"]), membership.get(e["to"])
+        if a and b and a != b:
+            counts[a, b, e["kind"]] += e["count"]
+    return [{"from": a, "to": b, "kind": k, "count": n,
+             **({"provenance": "model_inferred"} if k == "relation" else {})}
+            for (a, b, k), n in sorted(counts.items())]
+
+
+def relationship_view(tree: dict, edges: list[dict], graph: dict | None, *, graph_only_limit: int = 40,
+                      within_containment_limit: int = 40) -> dict:
+    """Compare disjoint subtrees; expose calls inside structural containment.
+
+    Tree containment pairs count ancestor direct units versus descendant subtree.
+    Graph-only pairs use direct primary ownership to avoid repeating edges at
+    every ancestor level. Only resolved call/import inputs are graph evidence.
+    """
+    from cbe.graph import graph_state
+
+    state, reason = graph_state(graph)
+    membership = {uid: n["id"] for n in tree.get("nodes", []) for uid in n.get("unit_ids", [])}
+    resolved = [e for e in edges if e["kind"] in {"call", "import"}
+                and e.get("provenance") != "model_inferred"] if state == "provided" else []
+    nodes = {n["id"]: n for n in tree.get("nodes", [])}
+    ancestors = {}
+    for nid in nodes:
+        lineage, parent = set(), nodes[nid].get("parent")
+        while parent in nodes and parent not in lineage:
+            lineage.add(parent)
+            parent = nodes[parent].get("parent")
+        ancestors[nid] = lineage
+    def contains(a, b):
+        return a in ancestors[b] or b in ancestors[a]
+    subtree_units = {nid: {uid for uid, owner in membership.items()
+                           if owner == nid or nid in ancestors[owner]} for nid in nodes}
+    counts = Counter()
+    kinds = defaultdict(Counter)
+    for edge in roll_up_edges(resolved, membership):
+        pair = edge["from"], edge["to"]
+        counts[pair] += edge["count"]
+        kinds[pair][edge["kind"]] += edge["count"]
+    node_ids = set(nodes)
+    model = {(e["from"], e["to"]): e.get("basis", "") for e in tree.get("depends_on", [])
+             if e.get("from") in node_ids and e.get("to") in node_ids and e["from"] != e["to"]}
+    unmentioned = counts.keys() - model.keys()
+    graph_only = sorted((p for p in unmentioned if not contains(*p)), key=lambda p: (-counts[p], p))
+    within_containment = sorted((p for p in unmentioned if contains(*p)), key=lambda p: (-counts[p], p))
+    reverse_counts = {}
+    inferred_counts = {}
+    relations = [e for e in edges if e["kind"] == "relation"]
+    for a, b in model:
+        if contains(a, b):
+            ancestor, descendant = (a, b) if a in ancestors[b] else (b, a)
+            pair_membership = {uid: ancestor for uid in nodes[ancestor].get("unit_ids", [])}
+            pair_membership.update({uid: descendant for uid in subtree_units[descendant]})
+        else:
+            pair_membership = {uid: nid for nid in (a, b) for uid in subtree_units[nid]}
+        rolled = roll_up_edges(resolved, pair_membership)
+        pair_kinds = Counter()
+        reverse_counts[a, b] = sum(edge["count"] for edge in rolled
+                                   if (edge["from"], edge["to"]) == (b, a))
+        for edge in rolled:
+            if (edge["from"], edge["to"]) == (a, b):
+                pair_kinds[edge["kind"]] += edge["count"]
+        counts[a, b] = sum(pair_kinds.values())
+        kinds[a, b] = pair_kinds
+        inferred_counts[a, b] = sum(edge["count"] for edge in roll_up_edges(relations, pair_membership)
+                                    if (edge["from"], edge["to"]) == (a, b))
+    pairs = sorted(model) + graph_only[:graph_only_limit] + within_containment[:within_containment_limit]
+    dependencies = [{"from": a, "to": b,
+                     "source": "containment" if contains(a, b) and (a, b) in model else
+                               "graph_confirmed" if counts[a, b] and (a, b) in model else
+                               "model_inferred" if (a, b) in model else
+                               "within_containment" if contains(a, b) else "graph_only",
+                     "count": counts[a, b],
+                     **({"reverse_count": reverse_counts[a, b]} if contains(a, b) and (a, b) in model else {}),
+                     "inferred_count": inferred_counts.get((a, b), 0),
+                     "containment": contains(a, b), "kinds": dict(sorted(kinds[a, b].items())),
+                     "basis": model.get((a, b), ""), "description": ""} for a, b in pairs]
+    totals = {source: sum(e["source"] == source for e in dependencies)
+              for source in ("graph_confirmed", "model_inferred", "graph_only")}
+    totals["graph_only"] = len(graph_only)
+    return {"dependencies": dependencies, "counts": totals,
+            "within_containment_count": len(within_containment),
+            "within_containment_omitted": max(0, len(within_containment) - within_containment_limit),
+            "graph_only_omitted": max(0, len(graph_only) - graph_only_limit),
+            "graph_state": state, "graph_reason": reason,
+            "candidate_flows": candidate_flows(resolved)}
+
+
+def candidate_flows(edges: list[dict], *, max_entries: int = 12, max_depth: int = 6,
+                    max_flows: int = 24) -> list[list[str]]:
+    """Stable bounded DFS from zero-incoming units, ranked by outgoing count.
+
+    Depth counts units (six steps). Cycles end at the last unique unit. Pure
+    cyclic components have no entry and intentionally yield no candidates.
+    """
+    adjacency = defaultdict(Counter)
+    incoming = set()
+    for e in edges:
+        if e["kind"] == "call" and e.get("provenance") != "model_inferred" and e["from"] != e["to"]:
+            adjacency[e["from"]][e["to"]] += e["count"]
+            incoming.add(e["to"])
+    entries = sorted(adjacency.keys() - incoming,
+                     key=lambda uid: (-sum(adjacency[uid].values()), uid))[:max_entries]
+    result = []
+    def visit(path):
+        if len(result) >= max_flows:
+            return
+        targets = sorted((uid for uid in adjacency[path[-1]] if uid not in path),
+                         key=lambda uid: (-adjacency[path[-1]][uid], uid))
+        if len(path) >= max_depth or not targets:
+            if len(path) > 1:
+                result.append(path)
+            return
+        for uid in targets:
+            if len(result) >= max_flows:
+                break
+            visit(path + [uid])
+    for uid in entries:
+        visit([uid])
+    return result
+
+
+def annotate_relationships(view: dict, system: dict, edges: list[dict]) -> dict:
+    """Publish model prose separately from the program's evidence labels."""
+    descriptions = {(e["from"], e["to"]): e["description"]
+                    for e in system.get("dependency_descriptions", [])}
+    calls = {(e["from"], e["to"]) for e in edges if e["kind"] == "call"
+             and e.get("provenance") != "model_inferred"} if view["graph_state"] == "provided" else set()
+    flows = []
+    for flow in system.get("key_flows", []):
+        steps = flow["steps"]
+        flows.append({**flow, "inferred_steps": [i for i in range(1, len(steps))
+                                                if (steps[i-1], steps[i]) not in calls]})
+    return {**view, "dependencies": [{**e, "description": descriptions.get((e["from"], e["to"]), "")}
+                                      for e in view["dependencies"]], "key_flows": flows}

cbe/api_completion.py

@@ -0,0 +1,410 @@
+"""OpenAI-compatible HTTP completion client for the CBE script driver.
+
+This is a thin, tool-free client for any endpoint that speaks the OpenAI
+``POST {base_url}/chat/completions`` shape (OpenAI, Azure OpenAI-compatible
+gateways, Command Code, local vLLM/Ollama OpenAI shims). It deliberately does
+not touch the product's system provider entry or any agent harness; it is the
+completion call of the ``generate.py`` driver, and it returns a
+:class:`Completion` record that the shared state, retry and usage accounting
+consume.
+
+Secrets: the API key is read from an environment variable (default
+``OPENAI_API_KEY``) or from a standard config file path. The key is never
+written to the request log, the raw archive, the CSV row or any error string,
+and an error body that happens to reflect it is redacted. Endpoint and model
+are both configurable; usage is reported when the endpoint returns it and left
+unknown otherwise.
+"""
+
+from __future__ import annotations
+
+import csv
+import json
+import os
+import random
+import re
+import time
+import urllib.error
+import urllib.request
+from dataclasses import dataclass
+from email.utils import parsedate_to_datetime
+from pathlib import Path
+from threading import Lock
+from typing import Any
+
+from cbe.checks import json_object
+
+from cbe.model_settings import ApiConfigError, DEFAULT_API_CONFIG, OPTIONAL_REQUEST_KEYS, load_config
+
+
+#: calls.csv columns; unknown usage stays an empty field, never zero.
+FIELDS = ("task", "attempt", "input_tokens", "output_tokens", "cache_tokens", "result", "seconds")
+CSV_LOCK = Lock()
+
+
+@dataclass
+class Completion:
+    value: dict[str, Any] | None
+    error: str | None
+    raw_text: str
+    usage: dict[str, int | None] | None
+    seconds: float
+
+
+def _csv_row(path: Path, row: dict[str, Any]) -> None:
+    with CSV_LOCK:
+        first = not path.exists()
+        with path.open("a", encoding="utf-8", newline="") as stream:
+            writer = csv.DictWriter(stream, fieldnames=FIELDS)
+            if first:
+                writer.writeheader()
+            writer.writerow({field: row.get(field, "") for field in FIELDS})
+
+
+#: HTTP 429, 502, 503 and 504 are provider pushback or a gateway hiccup, not answers: the call waits and repeats in place.
+#: A repeat does not use up an answer attempt; each one is a ``rate_limited`` row in calls.csv.
+RATE_LIMIT_STATUSES = (429, 502, 503, 504)
+RATE_LIMIT_RETRIES = 6
+BACKOFF_START_SECONDS = 5
+BACKOFF_MAX_SECONDS = 120
+RATE_LIMIT_BODY_CHARS = 300
+_sleep = time.sleep
+
+
+def _retry_delay(retry_after: str | None, retry: int) -> float:
+    """Seconds to wait before repeat number ``retry`` (from 0): the server's
+    ``Retry-After`` when it gives one, else 5 s doubling to 120 s; both with jitter."""
+    seconds: float | None = None
+    if retry_after:
+        try:
+            seconds = float(retry_after)
+        except ValueError:
+            try:
+                seconds = parsedate_to_datetime(retry_after).timestamp() - time.time()
+            except (TypeError, ValueError):
+                seconds = None
+    if seconds is None:
+        seconds = BACKOFF_START_SECONDS * 2 ** retry
+    seconds = min(max(seconds, 0.0), BACKOFF_MAX_SECONDS)
+    return min(seconds + random.uniform(0, max(1.0, seconds * 0.2)), BACKOFF_MAX_SECONDS)
+
+
+def _post(request: urllib.request.Request, timeout: int, *, task: str, attempt: int, csv_path: Path,
+          secrets: list[str], pushback: list[dict[str, Any]], clock: dict[str, float]) -> tuple[int, bytes]:
+    """POST once, repeating in place after HTTP 429/502/503/504. ``pushback`` collects what each refusal said;
+    ``clock['started']`` is reset so a row's seconds cover its own request, not the wait before it."""
+    for retry in range(RATE_LIMIT_RETRIES + 1):
+        clock["started"] = time.monotonic()
+        try:
+            with urllib.request.urlopen(request, timeout=timeout) as response:
+                return response.status, response.read(MAX_RESPONSE_BYTES + 1)
+        except urllib.error.HTTPError as exc:
+            if exc.code not in RATE_LIMIT_STATUSES:
+                raise
+            try:
+                body = _redact(exc.read(2048).decode("utf-8", "replace"), secrets)[:RATE_LIMIT_BODY_CHARS]
+            except OSError:
+                body = ""
+            retry_after = exc.headers.get("Retry-After") if exc.headers else None
+            pushback.append({"status": exc.code, "retry_after": retry_after, "body": body})
+            if retry == RATE_LIMIT_RETRIES:
+                raise
+            delay = _retry_delay(retry_after, retry)
+            pushback[-1]["waited_seconds"] = round(delay, 1)
+            _csv_row(csv_path, {"task": task, "attempt": attempt, "result": "rate_limited",
+                                "seconds": round(time.monotonic() - clock["started"], 3)})
+            _sleep(delay)
+    raise AssertionError("unreachable")  # pragma: no cover
+
+
+MAX_RESPONSE_BYTES = 8 * 1024 * 1024
+#: Bounded copy of the provider response kept in the durable ``.api.json`` record.
+#: Larger bodies are explicitly truncated, never silently dropped.
+MAX_PRESERVED_RESPONSE_CHARS = 20000
+
+
+def _api_key(config: dict[str, Any]) -> str | None:
+    """Read the key from the environment first, then an optional file.
+
+    The value is returned to build the Authorization header only; callers must
+    never log it."""
+    env_name = config.get("api_key_env") or "OPENAI_API_KEY"
+    key = os.environ.get(env_name)
+    if key:
+        return key.strip()
+    key_file = config.get("api_key_file")
+    if key_file:
+        try:
+            return Path(key_file).read_text(encoding="utf-8").strip() or None
+        except OSError:
+            return None
+    return None
+
+
+def endpoint_url(base_url: str) -> str:
+    return base_url.rstrip("/") + "/chat/completions"
+
+
+def _safe_endpoint(base_url: str) -> str:
+    """A log-safe endpoint: scheme, host and path only.
+
+    A base URL can carry userinfo or a credential query; neither belongs in a
+    durable artifact."""
+    from urllib.parse import urlsplit
+
+    parts = urlsplit(base_url)
+    host = parts.hostname or ""
+    if parts.port:
+        host = f"{host}:{parts.port}"
+    return f"{parts.scheme}://{host}{parts.path}" if parts.scheme else parts.path
+
+
+def _redact(text: str, secrets: list[str]) -> str:
+    redacted = text
+    for secret in secrets:
+        if secret:
+            redacted = redacted.replace(secret, "[redacted]")
+    return redacted
+
+
+def _request_body(model: str, prompt: str, config: dict[str, Any]) -> dict[str, Any]:
+    body: dict[str, Any] = {
+        "model": model,
+        "messages": [{"role": "user", "content": prompt}],
+        "temperature": config.get("temperature", 0),
+    }
+    if config.get("output_max_tokens") is not None:
+        body["max_tokens"] = config["output_max_tokens"]
+    for key in OPTIONAL_REQUEST_KEYS:
+        if key in config and config[key] is not None:
+            body[key] = config[key]
+    return body
+
+
+def _usage(payload: dict[str, Any]) -> dict[str, int] | None:
+    """Map provider usage onto the product convention: ``input`` is non-cached
+    prompt tokens and ``cache`` is the cached portion.
+
+    OpenAI reports ``prompt_tokens`` as the total including cached tokens, so
+    subtracting the cached count avoids double counting it in delivery
+    accounting. The raw provider object is kept separately by the caller."""
+    usage = payload.get("usage")
+    if not isinstance(usage, dict):
+        return None
+    prompt = usage.get("prompt_tokens")
+    completion = usage.get("completion_tokens")
+    if type(prompt) is not int or type(completion) is not int:
+        return None
+    details = usage.get("prompt_tokens_details") or {}
+    cached = details.get("cached_tokens") if isinstance(details, dict) else None
+    if type(cached) is not int:
+        return {"input": prompt, "output": completion, "cache": None}
+    return {"input": max(prompt - cached, 0), "output": completion, "cache": cached}
+
+
+def _extract_text(payload: dict[str, Any]) -> str:
+    choices = payload.get("choices")
+    if not isinstance(choices, list) or not choices:
+        raise ValueError("response has no choices")
+    choice = choices[0]
+    if not isinstance(choice, dict):
+        raise ValueError("response choice is not an object")
+    message = choice.get("message")
+    if isinstance(message, dict):
+        content = message.get("content")
+        if isinstance(content, str) and content.strip():
+            return content
+        if isinstance(content, list):
+            # OpenAI multimodal-style content parts: keep the text parts.
+            parts = [part.get("text") for part in content
+                     if isinstance(part, dict) and isinstance(part.get("text"), str)]
+            joined = "".join(parts)
+            if joined.strip():
+                return joined
+    text = choice.get("text")
+    if isinstance(text, str) and text.strip():
+        return text
+    finish = choice.get("finish_reason")
+    raise ValueError(f"response choice has no text content (finish_reason={finish!r})")
+
+
+def _choice_summary(payload: dict[str, Any]) -> dict[str, Any]:
+    """A small, secret-free view of the first choice for durable evidence."""
+    choices = payload.get("choices")
+    if not isinstance(choices, list) or not choices or not isinstance(choices[0], dict):
+        return {"choices": len(choices) if isinstance(choices, list) else None}
+    choice = choices[0]
+    message = choice.get("message") if isinstance(choice.get("message"), dict) else {}
+    content = message.get("content")
+    tool_calls = message.get("tool_calls")
+    return {
+        "finish_reason": choice.get("finish_reason"),
+        "message_role": message.get("role"),
+        "content_type": type(content).__name__,
+        "content_chars": len(content) if isinstance(content, (str, list, dict)) else 0,
+        "content_nonempty": isinstance(content, str) and bool(content.strip()),
+        "reasoning_content_present": bool(message.get("reasoning_content")),
+        "tool_calls": len(tool_calls) if isinstance(tool_calls, list) else 0,
+        "refusal_present": message.get("refusal") is not None,
+    }
+
+
+def _classify_no_text(choice: dict[str, Any] | None) -> str:
+    """Classify an empty-text response from what the provider actually said."""
+    choice = choice or {}
+    finish = choice.get("finish_reason")
+    if finish == "length":
+        return "output_limit:provider hit the output token limit and returned no usable text"
+    if finish == "content_filter":
+        return "content_filtered:provider filtered the response"
+    if choice.get("tool_calls"):
+        return "tool_call_only:provider returned tool calls but no text; this host is tool-free"
+    return (f"no_content:response choice has no text (finish_reason={finish!r}, "
+            f"content_type={choice.get('content_type')!r})")
+
+
+def _provider_evidence(payload: Any, raw_body: str, secrets: list[str],
+                       body_truncated: bool) -> dict[str, Any]:
+    """A bounded, redacted copy of what the provider returned, parsed or not.
+
+    This is written before text extraction so a failed/empty/malformed reply is
+    still diagnosable; the tiny view never carries a key or credential."""
+    rendered = raw_body
+    if payload is not None:
+        try:
+            rendered = json.dumps(payload, ensure_ascii=False)
+        except (TypeError, ValueError):
+            rendered = raw_body
+    redacted = _redact(rendered, secrets)
+    truncated = len(redacted) > MAX_PRESERVED_RESPONSE_CHARS
+    return {"body_truncated": body_truncated, "truncated": truncated,
+            "original_chars": len(redacted),
+            "text": redacted[:MAX_PRESERVED_RESPONSE_CHARS]}
+
+
+def invoke(*, task: str, attempt: int, prompt: str, run_dir: Path,
+           model: str, config: Path, timeout: int, csv_path: Path,
+           overrides: dict[str, Any] | None = None) -> Completion:
+    safe_task = re.sub(r"[^a-zA-Z0-9_.-]", "_", task)
+    raw_dir = run_dir / "raw"
+    raw_dir.mkdir(parents=True, exist_ok=True)
+    clock = {"started": time.monotonic()}
+    pushback: list[dict[str, Any]] = []
+    error: str | None = None
+    text = ""
+    usage: dict[str, int] | None = None
+    provider_usage: dict[str, Any] | None = None
+    choice: dict[str, Any] | None = None
+    finish_reason: Any = None
+    provider_evidence: dict[str, Any] | None = None
+    actual_model: str | None = None
+    status: int | None = None
+    log_endpoint = ""
+    secrets: list[str] = []
+    settings: dict[str, Any] = {}
+    try:
+        settings = load_config(config, model)
+        if overrides:
+            settings.update(overrides)
+        base_url = settings.get("base_url")
+        if not isinstance(base_url, str) or not base_url:
+            raise ApiConfigError("API config needs a base_url")
+        if not model:
+            raise ApiConfigError("model is required for the API host")
+        log_endpoint = _safe_endpoint(base_url)
+        key = _api_key(settings)
+        request_target = endpoint_url(base_url)
+        body = json.dumps(_request_body(model, prompt, settings)).encode("utf-8")
+        headers = {"Content-Type": "application/json",
+                   "User-Agent": "codebase-explorer/api-completion"}
+        # An endpoint may need no authentication at all (a local proxy).  When
+        # no key is configured the Authorization header is omitted, and the
+        # server's real 401/429 is classified downstream; a dummy secret is
+        # never invented to force a header.
+        if key:
+            headers["Authorization"] = f"Bearer {key}"
+            secrets.append(key)
+        for name, value in (settings.get("extra_headers") or {}).items():
+            if isinstance(name, str) and isinstance(value, str):
+                headers[name] = value
+                secrets.append(value)
+        request = urllib.request.Request(request_target, data=body, headers=headers, method="POST")
+        status, raw = _post(request, timeout, task=task, attempt=attempt, csv_path=csv_path,
+                            secrets=secrets, pushback=pushback, clock=clock)
+        body_truncated = len(raw) > MAX_RESPONSE_BYTES
+        raw_body = raw[:MAX_RESPONSE_BYTES].decode("utf-8", "replace")
+        payload: Any = None
+        try:
+            payload = json.loads(raw_body)
+        except (ValueError, json.JSONDecodeError):
+            payload = None
+        # Preserve the bounded, redacted provider evidence and usage BEFORE text
+        # extraction, so an empty/malformed reply is still diagnosable.
+        provider_evidence = _provider_evidence(payload, raw_body, secrets, body_truncated)
+        if isinstance(payload, dict):
+            actual_model = payload.get("model")
+            raw_usage = payload.get("usage")
+            provider_usage = raw_usage if isinstance(raw_usage, dict) else None
+            usage = _usage(payload)
+            choice = _choice_summary(payload)
+            finish_reason = choice.get("finish_reason")
+            try:
+                text = _extract_text(payload)
+            except ValueError:
+                error = _classify_no_text(choice)
+            if finish_reason in {"length", "max_tokens", "max_output_tokens"} or body_truncated:
+                error = "output_limit:provider response was truncated; retry with a larger limit or smaller batches"
+        else:
+            error = f"invalid_json:provider body is not a JSON object ({len(raw_body)} chars)"
+    except ApiConfigError as exc:
+        error = f"api_config:{_redact(str(exc), secrets)}"
+    except urllib.error.HTTPError as exc:
+        status = exc.code
+        try:
+            detail = _redact(exc.read(2048).decode("utf-8", "replace"), secrets)[:160]
+        except OSError:
+            detail = ""
+        # Never echo an Authorization header; the body is a provider message
+        # that may reflect the key, so it is redacted first.
+        if exc.code == 401:
+            error = "provider_auth"
+        elif exc.code in RATE_LIMIT_STATUSES:
+            error = "provider_limit"  # only after every repeat in place was refused
+        else:
+            error = f"provider_error:http_{exc.code}:{detail}"
+    except urllib.error.URLError as exc:
+        error = f"api_network:{type(exc.reason).__name__ if exc.reason else 'URLError'}"
+    except TimeoutError:
+        error = "timeout"
+    except (ValueError, json.JSONDecodeError) as exc:
+        error = f"invalid_json:{_redact(str(exc), secrets)[:160]}"
+    except OSError as exc:
+        error = f"api_io:{type(exc).__name__}:{_redact(str(exc), secrets)[:120]}"
+    elapsed = time.monotonic() - clock["started"]
+
+    if error is None:
+        try:
+            value = json_object(text)
+        except (ValueError, json.JSONDecodeError) as exc:
+            error = f"invalid_json:{_redact(str(exc), secrets)[:160]}"
+    (raw_dir / f"{safe_task}-{attempt}.api.json").write_text(
+        json.dumps({"endpoint": log_endpoint, "model": model, "status": status,
+                    "actual_model": actual_model,
+                    "request_settings": {key: settings[key] for key in
+                                         ("output_max_tokens", "reasoning_effort", "response_format") if key in settings},
+                    "timeout": timeout,
+                    "finish_reason": finish_reason, "choice": choice,
+                    "usage": usage, "provider_usage": provider_usage,
+                    "provider_response": provider_evidence,
+                    "rate_limited": pushback,
+                    "usage_basis": "provider prompt_tokens minus cached_tokens maps to input; "
+                                   "cache holds the cached portion",
+                    "error": error, "seconds": round(elapsed, 3)},
+                   ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+    (raw_dir / f"{safe_task}-{attempt}.stdout").write_text(text, encoding="utf-8")
+    _csv_row(csv_path, {"task": task, "attempt": attempt,
+                        "input_tokens": usage["input"] if usage else "",
+                        "output_tokens": usage["output"] if usage else "",
+                        "cache_tokens": (usage["cache"] if usage and usage["cache"] is not None else ""),
+                        "result": error or "ok", "seconds": round(elapsed, 3)})
+    return Completion(value if error is None else None, error, text, usage, elapsed)