jevassert 0.1.0__py3-none-any.whl

jevassert/__init__.py

@@ -0,0 +1,3 @@
+"""jevassert — record/replay regression tests for Jev question packs."""
+
+__version__ = "0.1.0"

jevassert/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+raise SystemExit(main())

jevassert/cli.py

@@ -0,0 +1,529 @@
+"""CLI: ``jevassert record | check | compare``."""
+
+from __future__ import annotations
+
+import argparse
+import dataclasses
+import hashlib
+import json
+import sys
+from pathlib import Path
+from typing import Any
+
+from . import __version__
+from .client import JevClient, JevError
+from .compare import CompareResult
+from .compare import compare as compare_recordings
+from .metrics import (
+    INPUT_USD_PER_MTOK,
+    SMALL_N_ITEMS,
+    OverallMetrics,
+    ThresholdSuggestion,
+    build_items,
+    compute,
+    evaluate_gates,
+    suggest_threshold,
+)
+from .packs import Pack, PackError, load_pack
+from .report import render_junit, render_markdown
+from .runner import load_predictions, record, write_predictions
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = _build_parser()
+    args = parser.parse_args(argv)
+    try:
+        return int(args.func(args))
+    except (PackError, FileNotFoundError, ValueError, JevError) as exc:
+        print(f"jevassert: error: {exc}", file=sys.stderr)
+        return 2
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="jevassert",
+        description="Record/replay regression tests for Jev question packs.",
+    )
+    parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    record_parser = sub.add_parser(
+        "record", help="call Jev for every case and write a predictions JSONL"
+    )
+    record_parser.add_argument("pack", help="pack directory (containing pack.yaml + cases.jsonl)")
+    record_parser.add_argument("-o", "--out", default="predictions.jsonl")
+    record_parser.add_argument("--concurrency", type=int, default=8)
+    record_parser.add_argument(
+        "--limit", type=int, default=None, help="record only the first N cases"
+    )
+    record_parser.add_argument(
+        "--model",
+        default=None,
+        help="model version to record (default: pack.tested or jev-latest)",
+    )
+    record_parser.add_argument("--base-url", default=None, help="override TYPESAFE_BASE_URL")
+    record_parser.add_argument(
+        "--resume",
+        action="store_true",
+        help="skip cases already recorded successfully in --out (retry errors only)",
+    )
+    record_parser.add_argument(
+        "--rpm", type=int, default=0, help="pace requests per minute (0 = unlimited)"
+    )
+    record_parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="estimate tokens and cost from the pack without sending anything",
+    )
+    record_parser.add_argument(
+        "--shuffle-options",
+        type=int,
+        default=None,
+        metavar="SEED",
+        help="robustness pass: permute Choice option order with this seed",
+    )
+    record_parser.add_argument(
+        "--repeat",
+        type=int,
+        default=1,
+        metavar="N",
+        help="record N independent rounds (extra rounds go to FILE-r2.jsonl, ...); "
+        "prints the discordant-decision count across rounds",
+    )
+    record_parser.set_defaults(func=_cmd_record)
+
+    check_parser = sub.add_parser("check", help="evaluate a recording against the pack gates")
+    check_parser.add_argument("pack")
+    check_parser.add_argument("-p", "--predictions", default="predictions.jsonl")
+    check_parser.add_argument("--json", action="store_true", help="print machine-readable JSON")
+    check_parser.add_argument("--junit", default=None, help="write JUnit XML to this path")
+    check_parser.add_argument("--report", default=None, help="write a markdown report to this path")
+    check_parser.add_argument(
+        "--failures", action="store_true", help="list every mismatch after the summary"
+    )
+    check_parser.add_argument(
+        "--bootstrap",
+        type=int,
+        default=1000,
+        help="bootstrap resamples for confidence intervals (0 disables)",
+    )
+    check_parser.add_argument(
+        "--target-precision",
+        type=float,
+        default=None,
+        help="suggest the highest-coverage threshold that reaches this precision",
+    )
+    check_parser.add_argument(
+        "--partition",
+        choices=("all", "dev", "test"),
+        default="all",
+        help="evaluate only the dev or test half (deterministic hash split) — "
+        "tune thresholds on dev, verify on test",
+    )
+    check_parser.add_argument(
+        "--partition-seed", type=int, default=0, help="seed for the dev/test split"
+    )
+    check_parser.add_argument(
+        "--partition-ratio",
+        type=float,
+        default=0.5,
+        help="dev share of the deterministic split (default 0.5)",
+    )
+    check_parser.set_defaults(func=_cmd_check)
+
+    compare_parser = sub.add_parser("compare", help="paired comparison of two recordings")
+    compare_parser.add_argument("pack")
+    compare_parser.add_argument("--a", required=True, help="baseline predictions JSONL")
+    compare_parser.add_argument("--b", required=True, help="candidate predictions JSONL")
+    compare_parser.set_defaults(func=_cmd_compare)
+
+    return parser
+
+
+def _cmd_record(args: argparse.Namespace) -> int:
+    pack = load_pack(args.pack)
+    if args.dry_run:
+        cases, tokens = _estimate_tokens(pack, args.limit)
+        cost = tokens * INPUT_USD_PER_MTOK / 1_000_000
+        print(
+            f"dry-run: {cases} cases, ~{tokens} input tokens, ~${cost:.4f} "
+            "at $0.042/M input tokens (rough estimate, nothing sent)"
+        )
+        return 0
+    if args.repeat < 1:
+        print("jevassert: error: --repeat must be >= 1", file=sys.stderr)
+        return 2
+
+    client = JevClient(base_url=args.base_url)
+    base_out = Path(args.out)
+    total_errors = 0
+    try:
+        for round_number in range(1, args.repeat + 1):
+            round_out = _round_path(base_out, round_number)
+            resume_from = None
+            if args.resume and round_out.is_file():
+                resume_from = load_predictions(round_out)
+
+            def progress(done: int, total: int, current: int = round_number) -> None:
+                prefix = f"round {current}: " if args.repeat > 1 else ""
+                print(
+                    f"\rrecording {prefix}{done}/{total} cases",
+                    end="",
+                    file=sys.stderr,
+                    flush=True,
+                )
+
+            records = record(
+                pack,
+                client,
+                concurrency=args.concurrency,
+                limit=args.limit,
+                progress=progress,
+                model=args.model,
+                resume_from=resume_from,
+                rpm=args.rpm,
+                shuffle_options=args.shuffle_options,
+            )
+            print(file=sys.stderr)
+            write_predictions(round_out, records)
+            errors = sum(1 for item in records if item["error"])
+            total_errors += errors
+            print(
+                f"{len(records)} cases -> {round_out}" + (f" ({errors} errors)" if errors else "")
+            )
+            for item in records:
+                if item["error"]:
+                    print(f"  {item['case_id']}: {item['error']}", file=sys.stderr)
+    finally:
+        client.close()
+
+    if args.repeat > 1:
+        print(_stability_summary(pack, base_out, args.repeat))
+    return 1 if total_errors else 0
+
+
+def _round_path(base: Path, round_number: int) -> Path:
+    if round_number == 1:
+        return base
+    return base.with_name(f"{base.stem}-r{round_number}{base.suffix}")
+
+
+def _stability_summary(pack: Pack, base_out: Path, repeats: int) -> str:
+    base = load_predictions(_round_path(base_out, 1))
+    flips = 0
+    compared = 0
+    for round_number in range(2, repeats + 1):
+        other_path = _round_path(base_out, round_number)
+        if not other_path.is_file():
+            continue
+        try:
+            result = compare_recordings(pack, base, load_predictions(other_path))
+        except ValueError:
+            continue
+        flips += result.wins + result.losses
+        compared = max(compared, result.n_items)
+    if not compared:
+        return "stability: no comparable rounds"
+    return (
+        f"stability: {flips} discordant decisions across {repeats} rounds "
+        f"({compared} items compared, {flips / compared:.3f} flip rate)"
+    )
+
+
+def _estimate_tokens(pack: Pack, limit: int | None) -> tuple[int, int]:
+    cases = pack.cases[:limit] if limit else pack.cases
+    question_tokens = len(json.dumps(pack.to_api_questions(), ensure_ascii=False)) // 4 + 1
+    tokens = sum(
+        len(json.dumps(case.state, ensure_ascii=False)) // 4 + question_tokens for case in cases
+    )
+    return len(cases), tokens
+
+
+def _cmd_check(args: argparse.Namespace) -> int:
+    pack = load_pack(args.pack)
+    predictions = load_predictions(args.predictions)
+    _warn_on_recording_mismatch(pack, predictions)
+    if args.partition != "all":
+        pack = _partition_pack(pack, args.partition, args.partition_seed, args.partition_ratio)
+        case_ids = {case.id for case in pack.cases}
+        predictions = {
+            case_id: item for case_id, item in predictions.items() if case_id in case_ids
+        }
+        print(
+            f"partition: {args.partition} — {len(pack.cases)} cases "
+            f"(seed {args.partition_seed}, ratio {args.partition_ratio})",
+            file=sys.stderr,
+        )
+    overall = compute(pack, predictions, bootstrap=args.bootstrap)
+    gates = evaluate_gates(pack, overall)
+    suggestion = None
+    if args.target_precision is not None:
+        items, _, _ = build_items(pack, predictions)
+        suggestion = suggest_threshold(items, args.target_precision)
+
+    if args.junit:
+        Path(args.junit).write_text(render_junit(pack, gates), encoding="utf-8")
+    if args.report:
+        Path(args.report).write_text(
+            render_markdown(pack, overall, gates, suggestion, args.target_precision),
+            encoding="utf-8",
+        )
+
+    if args.json:
+        print(json.dumps(_payload(pack, overall, gates, suggestion), indent=2, ensure_ascii=False))
+    else:
+        print(
+            _human_summary(
+                pack, overall, gates, args.predictions, suggestion, args.target_precision
+            )
+        )
+        if args.failures:
+            print()
+            print(_failures(pack, predictions))
+
+    return 1 if any(gate.ok is False for gate in gates) else 0
+
+
+def _warn_on_recording_mismatch(pack: Pack, predictions: dict[str, dict[str, Any]]) -> None:
+    """Never let a stale or partial recording pass silently."""
+    pack_ids = {case.id for case in pack.cases}
+    extra = sorted(set(predictions) - pack_ids)
+    missing = sorted(pack_ids - set(predictions))
+    if extra:
+        print(
+            f"warning: recording has {len(extra)} case(s) not in the pack: {', '.join(extra[:5])}",
+            file=sys.stderr,
+        )
+    if missing:
+        print(
+            f"warning: recording is missing {len(missing)} case(s): {', '.join(missing[:5])}",
+            file=sys.stderr,
+        )
+    models = {record.get("model") for record in predictions.values() if record.get("model")}
+    if pack.tested and models and models != {pack.tested}:
+        print(
+            f"warning: recording model(s) {sorted(models)} != pack.tested {pack.tested}",
+            file=sys.stderr,
+        )
+    if pack.tested is None:
+        print(
+            "note: pack.tested is null — pin it to the recorded model version "
+            "once evidence is reviewed",
+            file=sys.stderr,
+        )
+
+
+def _partition_pack(pack: Pack, which: str, seed: int, ratio: float) -> Pack:
+    """Deterministic dev/test half so thresholds are tuned and verified apart."""
+    selected = tuple(case for case in pack.cases if _in_partition(case.id, which, seed, ratio))
+    return dataclasses.replace(pack, cases=selected)
+
+
+def _in_partition(case_id: str, which: str, seed: int, ratio: float) -> bool:
+    digest = hashlib.blake2s(f"{seed}:{case_id}".encode(), digest_size=8).digest()
+    is_dev = int.from_bytes(digest, "big") / 2**64 < ratio
+    return is_dev if which == "dev" else not is_dev
+
+
+def _failures(pack: Pack, predictions: dict[str, dict[str, Any]]) -> str:
+    items, _, _ = build_items(pack, predictions)
+    misses = [item for item in items if not item.correct]
+    states = {case.id: case.state for case in pack.cases}
+    lines = [f"failures: {len(misses)}/{len(items)} items"]
+    for item in misses:
+        lines.append(
+            f"  {item.case_id} {item.qid}: expected={item.expected!r} got={item.got!r} "
+            f"p={item.decision_prob:.2f}"
+        )
+        lines.append(f"    state: {json.dumps(states[item.case_id], ensure_ascii=False)[:200]}")
+    return "\n".join(lines)
+
+
+def _cmd_compare(args: argparse.Namespace) -> int:
+    pack = load_pack(args.pack)
+    result = compare_recordings(pack, load_predictions(args.a), load_predictions(args.b))
+    print(_compare_summary(pack, result, args.a, args.b))
+    return 0
+
+
+def _payload(
+    pack: Pack,
+    overall: OverallMetrics,
+    gates: list[Any],
+    suggestion: ThresholdSuggestion | None = None,
+) -> dict[str, Any]:
+    return {
+        "pack": pack.id,
+        "version": pack.version,
+        "record_model": pack.record_model,
+        "tested": pack.tested,
+        "n_cases": overall.n_cases,
+        "n_items": overall.n_items,
+        "case_errors": overall.case_errors,
+        "missing_items": overall.missing_items,
+        "accuracy": overall.accuracy,
+        "accuracy_ci": list(overall.accuracy_ci) if overall.accuracy_ci else None,
+        "ece": overall.ece,
+        "ece_ci": list(overall.ece_ci) if overall.ece_ci else None,
+        "suggestion": (
+            {
+                "threshold": suggestion.threshold,
+                "coverage": suggestion.coverage,
+                "precision": suggestion.precision,
+                "n_accepted": suggestion.n_accepted,
+            }
+            if suggestion
+            else None
+        ),
+        "mean_decision_prob": overall.mean_decision_prob,
+        "cost_per_case_usd": overall.cost_per_case_usd,
+        "total_cost_usd": overall.total_cost_usd,
+        "p50_latency_ms": overall.p50_latency_ms,
+        "p95_latency_ms": overall.p95_latency_ms,
+        "per_question": {
+            qid: {
+                "type": metrics.qtype,
+                "n": metrics.n,
+                "missing": metrics.missing,
+                "accuracy": metrics.accuracy,
+                "mean_decision_prob": metrics.mean_decision_prob,
+                "ece": metrics.ece,
+                "brier": metrics.brier,
+            }
+            for qid, metrics in overall.per_question.items()
+        },
+        "coverage": [
+            {
+                "threshold": row.threshold,
+                "coverage": row.coverage,
+                "precision": row.precision,
+            }
+            for row in overall.coverage
+        ],
+        "threshold_coverage": (
+            {
+                "n_total": overall.threshold_coverage.n_total,
+                "n_auto": overall.threshold_coverage.n_auto,
+                "coverage": overall.threshold_coverage.coverage,
+                "precision": overall.threshold_coverage.precision,
+            }
+            if overall.threshold_coverage
+            else None
+        ),
+        "gates": [{"gate": gate.gate, "ok": gate.ok, "detail": gate.detail} for gate in gates],
+    }
+
+
+def _human_summary(
+    pack: Pack,
+    overall: OverallMetrics,
+    gates: list[Any],
+    predictions_path: str,
+    suggestion: ThresholdSuggestion | None = None,
+    target_precision: float | None = None,
+) -> str:
+    lines: list[str] = []
+    tested = f", tested {pack.tested}" if pack.tested else ", provisional"
+    lines.append(
+        f"jevassert — {pack.id} v{pack.version} (record model {pack.record_model}{tested})"
+    )
+    lines.append(
+        f"recording: {predictions_path} — {overall.n_cases} cases, "
+        f"{overall.case_errors} errors, {overall.missing_items} missing, {overall.n_items} items"
+    )
+    lines.append("")
+    lines.append(
+        f"{'question':<22} {'type':<7} {'n':>3} {'acc':>7} {'p(dec)':>7} {'ece':>7} {'brier':>7}"
+    )
+    for qid, metrics in overall.per_question.items():
+        brier = f"{metrics.brier:.3f}" if metrics.brier is not None else "—"
+        lines.append(
+            f"{qid:<22} {metrics.qtype:<7} {metrics.n:>3} {metrics.accuracy:>7.3f} "
+            f"{metrics.mean_decision_prob:>7.3f} {metrics.ece:>7.3f} {brier:>7}"
+        )
+    lines.append(
+        f"{'overall':<22} {'':<7} {overall.n_items:>3} {overall.accuracy:>7.3f} "
+        f"{overall.mean_decision_prob:>7.3f} {overall.ece:>7.3f}"
+    )
+    if overall.accuracy_ci is not None:
+        low, high = overall.accuracy_ci
+        ece_part = ""
+        if overall.ece_ci is not None:
+            ece_part = f", ECE CI {overall.ece_ci[0]:.3f}–{overall.ece_ci[1]:.3f}"
+        lines.append(f"bootstrap 95%: accuracy CI {low:.3f}–{high:.3f}{ece_part}")
+    if overall.n_items < SMALL_N_ITEMS:
+        lines.append(
+            f"note: {overall.n_items} items only — accuracy CI is wide and ECE is coarse "
+            "(see README: Reading the numbers)"
+        )
+    if overall.cost_per_case_usd is not None:
+        lines.append(
+            f"cost/case ${overall.cost_per_case_usd:.6f} "
+            f"(total ${overall.total_cost_usd:.4f} at ${0.042}/M input tokens)"
+        )
+    if overall.p95_latency_ms is not None:
+        lines.append(
+            f"latency p50 {overall.p50_latency_ms:.0f}ms, p95 {overall.p95_latency_ms:.0f}ms"
+        )
+    lines.append("")
+    lines.append("coverage (accept when p >= threshold):")
+    for row in overall.coverage:
+        precision = f"{row.precision:.3f}" if row.precision is not None else "—"
+        lines.append(
+            f"  >= {row.threshold:.2f}: coverage {row.coverage:.3f}, precision {precision}"
+        )
+    if overall.threshold_coverage is not None:
+        tc = overall.threshold_coverage
+        precision = f"{tc.precision:.3f}" if tc.precision is not None else "—"
+        lines.append("")
+        lines.append("author thresholds (pack floors; labels without a floor go to review):")
+        lines.append(
+            f"  auto-accepted {tc.n_auto}/{tc.n_total} ({tc.coverage:.3f}), precision {precision}"
+        )
+    if suggestion is not None and target_precision is not None:
+        lines.append("")
+        lines.append(
+            f"threshold suggestion (precision >= {target_precision:.2f}): "
+            f"p >= {suggestion.threshold:.2f} → coverage {suggestion.coverage:.3f}, "
+            f"precision {suggestion.precision:.3f} (n={suggestion.n_accepted})"
+        )
+    lines.append("")
+    lines.append("gates:")
+    if not gates:
+        lines.append("  (none declared in pack.yaml)")
+    for gate in gates:
+        mark = "PASS" if gate.ok else ("SKIP" if gate.ok is None else "FAIL")
+        lines.append(f"  {mark} {gate.gate}: {gate.detail}")
+    return "\n".join(lines)
+
+
+def _compare_summary(pack: Pack, result: CompareResult, path_a: str, path_b: str) -> str:
+    lines: list[str] = []
+    lines.append(f"jevassert compare — {pack.id} v{pack.version}")
+    lines.append(f"  A: {path_a}")
+    lines.append(f"  B: {path_b}")
+    lines.append(
+        f"  items {result.n_items}: accuracy A {result.accuracy_a:.3f} "
+        f"-> B {result.accuracy_b:.3f} (delta {result.delta:+.3f})"
+    )
+    lines.append(
+        f"  discordant: A-only correct {result.wins}, B-only correct {result.losses} "
+        f"— McNemar exact p = {result.p_value:.4f}"
+    )
+    if result.cost_per_case_a_usd is not None and result.cost_per_case_b_usd is not None:
+        lines.append(
+            f"  cost/case A ${result.cost_per_case_a_usd:.6f} "
+            f"-> B ${result.cost_per_case_b_usd:.6f}"
+        )
+    lines.append("")
+    lines.append(f"{'question':<22} {'n':>3} {'acc A':>7} {'acc B':>7} {'delta':>7}")
+    for delta in result.per_question.values():
+        lines.append(
+            f"{delta.qid:<22} {delta.n:>3} {delta.accuracy_a:>7.3f} "
+            f"{delta.accuracy_b:>7.3f} {delta.delta:>+7.3f}"
+        )
+    return "\n".join(lines)
+
+
+if __name__ == "__main__":  # pragma: no cover
+    raise SystemExit(main())

jevassert/client.py

@@ -0,0 +1,148 @@
+"""HTTP transport for the TypeSafe System One endpoint.
+
+One small client with retries and an injectable transport so tests run
+without network access and so any Jev-compatible endpoint (TypeSafe API,
+gateway, local replica) can be used via ``base_url``.
+"""
+
+from __future__ import annotations
+
+import os
+import random
+import time
+from collections.abc import Callable
+from typing import Any, Protocol
+
+import httpx
+
+from . import __version__
+
+DEFAULT_BASE_URL = "https://api.typesafe.ai"
+ENDPOINT = "/v1/systemone"
+RETRYABLE_STATUS = {429, 500, 502, 503, 504, 529}
+
+# transport(method, url, headers, json_body) -> (status, body, retry_after_seconds)
+Transport = Callable[[str, str, dict[str, str], dict[str, Any]], tuple[int, Any, float | None]]
+
+
+class JevError(RuntimeError):
+    """A System One call failed (after retries, if retryable)."""
+
+    def __init__(self, message: str, status: int | None = None) -> None:
+        super().__init__(message)
+        self.status = status
+
+    @property
+    def is_auth_error(self) -> bool:
+        return self.status in (401, 403)
+
+
+class _ResponseLike(Protocol):  # pragma: no cover - typing helper
+    status_code: int
+    headers: Any
+
+    def json(self) -> Any: ...
+
+
+def _httpx_transport(client: httpx.Client) -> Transport:
+    def transport(
+        method: str, url: str, headers: dict[str, str], json_body: dict[str, Any]
+    ) -> tuple[int, Any, float | None]:
+        response = client.request(method, url, headers=headers, json=json_body)
+        retry_after: float | None = None
+        raw = response.headers.get("retry-after")
+        if raw:
+            try:
+                retry_after = float(raw)
+            except ValueError:
+                retry_after = None
+        try:
+            body = response.json()
+        except ValueError:
+            body = response.text
+        return response.status_code, body, retry_after
+
+    return transport
+
+
+class JevClient:
+    """Thin client for ``POST {base_url}/v1/systemone``.
+
+    Reads ``TYPESAFE_API_KEY`` and ``TYPESAFE_BASE_URL`` from the environment
+    when not given explicitly. The API key is never logged or written to disk.
+    """
+
+    def __init__(
+        self,
+        api_key: str | None = None,
+        base_url: str | None = None,
+        transport: Transport | None = None,
+        timeout: float = 60.0,
+        max_retries: int = 3,
+        backoff_seconds: float = 0.5,
+    ) -> None:
+        self.api_key = api_key or os.environ.get("TYPESAFE_API_KEY")
+        self.base_url = (
+            base_url or os.environ.get("TYPESAFE_BASE_URL") or DEFAULT_BASE_URL
+        ).rstrip("/")
+        self.max_retries = max_retries
+        self.backoff_seconds = backoff_seconds
+        self._http = None if transport else httpx.Client(timeout=timeout)
+        self._transport: Transport = transport or _httpx_transport(self._http)  # type: ignore[arg-type]
+
+    def __enter__(self) -> JevClient:
+        return self
+
+    def __exit__(self, *exc_info: object) -> None:
+        self.close()
+
+    def close(self) -> None:
+        if self._http is not None:
+            self._http.close()
+            self._http = None
+
+    def system_one(
+        self, state: Any, questions: dict[str, Any], model: str = "jev-latest"
+    ) -> tuple[dict[str, Any], float]:
+        """Evaluate ``questions`` against ``state``; returns (response, latency_ms)."""
+        if not self.api_key:
+            raise JevError("TYPESAFE_API_KEY is not set (env var or api_key=)")
+        payload = {"state": state, "model": model, "questions": questions}
+        headers = {
+            "Authorization": f"Bearer {self.api_key}",
+            "Content-Type": "application/json",
+            "User-Agent": f"jevassert/{__version__}",
+        }
+        url = f"{self.base_url}{ENDPOINT}"
+
+        for attempt in range(self.max_retries + 1):
+            started = time.perf_counter()
+            try:
+                status, body, retry_after = self._transport("POST", url, headers, payload)
+            except httpx.TransportError as exc:
+                if attempt >= self.max_retries:
+                    raise JevError(f"request failed after {attempt + 1} attempts: {exc}") from exc
+                self._sleep(attempt, None)
+                continue
+            latency_ms = (time.perf_counter() - started) * 1000
+
+            if status == 200 and isinstance(body, dict):
+                return body, latency_ms
+            if status in RETRYABLE_STATUS and attempt < self.max_retries:
+                self._sleep(attempt, retry_after)
+                continue
+            raise JevError(f"HTTP {status}: {_excerpt(body)}", status=status)
+
+        raise JevError("unreachable")  # pragma: no cover
+
+    def _sleep(self, attempt: int, retry_after: float | None) -> None:
+        delay = self.backoff_seconds * (2**attempt) + random.uniform(0, 0.25)
+        if retry_after:
+            delay = max(delay, retry_after)
+        time.sleep(delay)
+
+
+def _excerpt(body: Any, limit: int = 300) -> str:
+    text = body if isinstance(body, str) else str(body)
+    text = " ".join(text.split())
+    return text[:limit] + ("…" if len(text) > limit else "")