otto-cli-agent 0.1.0__py3-none-any.whl

agent/cli/eval.py

@@ -0,0 +1,130 @@
+"""`otto eval`: run the golden dataset through the real pipeline and report
+pass/fail per item. See agent/eval/runner.py for what "checked" means, and
+agent/eval/golden/ for the items themselves.
+
+By default this tracks the run as a Langfuse Dataset Run (agent/eval/
+langfuse_sync.py): the golden set is synced to a Langfuse Dataset once,
+then every `otto eval` invocation becomes one named, comparable run
+against it in the Langfuse UI -- every item's trace, its pass/fail score,
+and the run as a whole. Pass --no-experiment for the old local-only path
+(no Langfuse project needed -- useful offline or in CI without Langfuse
+creds), which prints a plain pass/fail table and nothing else.
+
+No --agents option (2026-09-10): the router/planner/solver/summarizer/
+finder/evaluator graph that replaced the swarm pipeline has nothing to
+size -- see agent/pipeline/run.py's module docstring.
+"""
+from typing import Annotated, Optional
+
+import typer
+from rich import box
+from rich.table import Table
+
+from agent.cli.ui import err, out
+from agent.eval.runner import run_golden
+
+
+def eval_cmd(
+    domain: Annotated[Optional[str], typer.Option(help="Only 'code' or 'math'.")] = None,
+    experiment: Annotated[
+        bool,
+        typer.Option(
+            "--experiment/--no-experiment",
+            help="Track this run as a Langfuse Dataset Run (default) vs. a local-only table.",
+        ),
+    ] = True,
+    run_name: Annotated[
+        Optional[str], typer.Option(help="Name for the Langfuse experiment run (--experiment only).")
+    ] = None,
+) -> None:
+    """Run the golden dataset through the pipeline and report pass/fail."""
+    if domain is not None and domain not in ("code", "math"):
+        err.print("[bad]--domain must be 'code' or 'math'[/]")
+        raise typer.Exit(2)
+
+    if experiment:
+        _run_as_langfuse_experiment(domain=domain, run_name=run_name)
+        return
+
+    with err.status("running golden set…"):
+        results = run_golden(only_domain=domain)
+
+    if not results:
+        out.print("[muted]no golden items matched[/]")
+        return
+
+    t = Table(box=box.SIMPLE, header_style="muted")
+    t.add_column("id", style="spec")
+    t.add_column("domain", style="muted")
+    t.add_column("result")
+    t.add_column("seconds", justify="right")
+    t.add_column("evidence", style="muted")
+    passed = 0
+    for r in results:
+        mark = "[ok]pass[/]" if r.passed else "[bad]fail[/]"
+        passed += r.passed
+        evidence = r.evidence if len(r.evidence) <= 80 else r.evidence[:80] + "…"
+        t.add_row(r.item_id, r.domain, mark, f"{r.seconds:.1f}", evidence.replace("\n", " "))
+    out.print(t)
+    out.print(f"[muted]{passed}/{len(results)} passed[/]")
+
+    _print_full_failures((r.item_id, r.evidence) for r in results if not r.passed)
+    if passed < len(results):
+        raise typer.Exit(1)
+
+
+def _print_full_failures(failures) -> None:
+    """The overview table truncates evidence to 80 chars for readability --
+    fine for a scan, useless for actually debugging a failure (it cuts a
+    checker's stderr right where the real exception would start). Print
+    each failing item's full evidence separately so that's never the
+    reason you have to go dig through the Langfuse UI.
+    """
+    failures = list(failures)
+    if not failures:
+        return
+    out.print("\n[bad]failures, in full:[/]")
+    for item_id, evidence in failures:
+        out.print(f"[spec]{item_id}[/]")
+        out.print(evidence)
+        out.print("")
+
+
+def _run_as_langfuse_experiment(*, domain: Optional[str], run_name: Optional[str]) -> None:
+    from agent.eval.langfuse_sync import run_golden_experiment
+
+    with err.status("syncing golden set to langfuse and running experiment…"):
+        try:
+            result = run_golden_experiment(only_domain=domain, run_name=run_name)
+        except Exception as exc:
+            err.print(f"[bad]langfuse experiment failed: {exc}[/]")
+            err.print("[muted]is LANGFUSE_PUBLIC_KEY / LANGFUSE_SECRET_KEY set? falling back to --no-experiment still works offline.[/]")
+            raise typer.Exit(1) from exc
+
+    t = Table(box=box.SIMPLE, header_style="muted")
+    t.add_column("id", style="spec")
+    t.add_column("result")
+    t.add_column("evidence", style="muted")
+    all_passed = True
+    for item_result in result.item_results:
+        golden_id = getattr(item_result.item, "id", "?")
+        ev = next((e for e in item_result.evaluations if e.name == "golden_pass"), None)
+        item_passed = bool(ev.value) if ev is not None else False
+        all_passed &= item_passed
+        mark = "[ok]pass[/]" if item_passed else "[bad]fail[/]"
+        comment = (ev.comment or "") if ev is not None else "no golden_pass score"
+        comment = comment if len(comment) <= 80 else comment[:80] + "…"
+        t.add_row(golden_id, mark, comment.replace("\n", " "))
+    out.print(t)
+
+    n = len(result.item_results)
+    n_passed = sum(
+        1 for r in result.item_results
+        if any(e.name == "golden_pass" and e.value for e in r.evaluations)
+    )
+    out.print(f"[muted]{n_passed}/{n} passed[/]")
+    if result.dataset_run_url:
+        out.print(f"[muted]langfuse dataset run: {result.dataset_run_url}[/]")
+
+    if not all_passed:
+        raise typer.Exit(1)

agent/cli/eval_claw.py

@@ -0,0 +1,414 @@
+"""`otto eval-claw` -- Claw-Eval's 300 tasks against Otto's own agent.
+
+Their `claw-eval run` can only point at a model endpoint, so the number it
+gives is about a model inside their agent loop. This command runs OTTO on the
+task -- its graph, its memory, its tools, its evaluator -- and hands the
+resulting trace to their own graders, so the score is about the agent.
+See agent/eval/claw_bench.py for the seam that makes that possible.
+
+Needs a Claw-Eval checkout (--claw-root or $CLAW_EVAL_ROOT) with its
+requirements installed, and Docker for the 169 tasks whose files live in a
+container. Start with --limit: every task is a full agent run.
+"""
+from __future__ import annotations
+
+import json
+import time
+from contextlib import ExitStack
+from pathlib import Path
+from types import SimpleNamespace
+from typing import Optional
+
+import typer
+from typing_extensions import Annotated
+
+from agent.cli.ui import err, out
+from agent.eval import failures
+from agent.memory.lessons import bind_bank, read_only
+from agent.memory.store import MemoryStore
+from agent.router.outcomes import bind_log
+from agent.router.outcomes import read_only as routing_read_only
+from agent.eval.claw_bench import (
+    ClawEvalUnavailable,
+    claw_root,
+    grading_fingerprint,
+    load_claw,
+    missing_service_keys,
+    run_task_file,
+    select_tasks,
+    split_tasks,
+)
+
+
+#: How many trials per task before a Claw-Eval number is evidence of
+#: anything.
+#:
+#: Measured, not chosen. Task T093 on identical code and configuration, three
+#: runs: 0.86, 0.60, 0.96 -- a spread of 0.36 on a 0-1 scale. Over two trials
+#: pass^2 came out 0.25 against pass@2 of 1.00: it got there once and not
+#: twice. Completion is LLM-judged for 260 of the 300 tasks, which is where
+#: most of that comes from.
+#:
+#: The consequence is retroactive and worth stating plainly: any single-run
+#: before/after difference smaller than about 0.3 on this benchmark is inside
+#: the noise. Much of the optimisation work on this branch was read from one
+#: run per task, and the only difference in that set large enough to survive
+#: was a crash going from 0.00 to 0.955.
+#:
+#: Three, matching what the issues here already ask for when they say how to
+#: settle something ("--trials 3").
+MIN_TRIALS_FOR_EVIDENCE = 3
+
+#: The observed spread above, quoted in the warning so the number a reader is
+#: being told to distrust comes with the reason.
+OBSERVED_SINGLE_RUN_SPREAD = 0.36
+
+
+def _summary(outcomes: list, claw=None) -> dict:
+    scored = [o for o in outcomes if o is not None]
+    if not scored:
+        return {"tasks": 0}
+    n = len(scored)
+    summary = {
+        "tasks": n,
+        "passed": sum(1 for o in scored if o.passed),
+        "pass_rate": round(sum(1 for o in scored if o.passed) / n, 4),
+        "mean_task_score": round(sum(o.task_score for o in scored) / n, 4),
+        "mean_completion": round(sum(o.completion for o in scored) / n, 4),
+        "mean_robustness": round(sum(o.robustness for o in scored) / n, 4),
+        "mean_communication": round(sum(o.communication for o in scored) / n, 4),
+        "errors": sum(1 for o in scored if o.error),
+        "mean_wall_time_s": round(sum(o.wall_time_s for o in scored) / n, 2),
+        # The cost axis. None of the 300 graders read it, so without this a
+        # change that doubles spend for a tenth of a point reads as a win.
+        "mean_model_calls": round(sum(o.model_calls for o in scored) / n, 2),
+        "total_model_calls": sum(o.model_calls for o in scored),
+    }
+
+    # Always present, so a reader of otto_summary.json never has to infer the
+    # trial count from whether a reliability key happens to exist.
+    summary["trials"] = min((len(o.trials) for o in scored), default=0)
+    #: Whether this run is enough trials to be read as a measurement at all --
+    #: MIN_TRIALS_FOR_EVIDENCE. False does NOT mean the numbers are wrong; it
+    #: means the difference between them and another run's is not attributable
+    #: to anything. Carried in the JSON as well as printed, because the JSON is
+    #: what gets pasted into a comparison months later.
+    summary["is_evidence"] = summary["trials"] >= MIN_TRIALS_FOR_EVIDENCE
+
+    # What KIND of failure, and what the tools cost -- agent/eval/failures.py,
+    # read off the action lines every run already produced, with no model
+    # call. A score says something got worse; "38% of the failures changed no
+    # files at all" says where to look.
+    failed = {o.task_id: getattr(o, "actions", None)
+              for o in scored if not o.passed}
+    if failed:
+        summary["failures"] = failures.summarise(failed)
+    summary["tool_cost"] = failures.total_tool_cost(
+        getattr(o, "actions", None) for o in scored
+    ).to_dict()
+
+    repeated = [o for o in scored if len(o.trials) > 1]
+    if repeated and claw is not None:
+        k = min(len(o.trials) for o in repeated)
+        summary["trials"] = k
+        # Per task first, then averaged: pass^k over a task's own repeats says
+        # "does it do this reliably", which is the property a self-improving
+        # loop is most able to fake by finding one lucky trial.
+        summary["mean_pass_hat_k"] = round(
+            sum(claw.compute_pass_hat_k(o.trials, k=k) for o in repeated) / len(repeated), 4
+        )
+        summary["mean_pass_at_k"] = round(
+            sum(claw.compute_pass_at_k(o.trials, k=k) for o in repeated) / len(repeated), 4
+        )
+        summary["score_spread"] = round(
+            sum(max(o.trials) - min(o.trials) for o in repeated) / len(repeated), 4
+        )
+    return summary
+
+
+def eval_claw_cmd(
+    claw_root_opt: Annotated[
+        Optional[Path],
+        typer.Option("--claw-root", help="Claw-Eval checkout; defaults to $CLAW_EVAL_ROOT."),
+    ] = None,
+    tag: Annotated[
+        Optional[str],
+        typer.Option(help="Capability to run: general, multimodal, user_agent, multi_service."),
+    ] = None,
+    task: Annotated[
+        Optional[str], typer.Option(help="Substring of a task directory name, e.g. 'T01' or 'M001'."),
+    ] = None,
+    limit: Annotated[Optional[int], typer.Option(help="Run at most this many tasks.")] = 5,
+    architecture: Annotated[
+        str, typer.Option(help="'graph' (the real pipeline) or 'single' (one-conversation control)."),
+    ] = "graph",
+    config: Annotated[
+        Optional[Path], typer.Option(help="Claw-Eval config.yaml (judge, sandbox, user-agent models)."),
+    ] = None,
+    trace_dir: Annotated[
+        Optional[Path], typer.Option(help="Where to write traces (default: <checkout>/traces/otto_<time>).")
+    ] = None,
+    no_judge: Annotated[bool, typer.Option("--no-judge", help="Skip the LLM judge.")] = False,
+    port_offset: Annotated[int, typer.Option(help="Shift every mock service port, for parallel runs.")] = 0,
+    max_seconds: Annotated[
+        Optional[float],
+        typer.Option(help="Cap each task's budget below its own (tasks allow 120-900s). "
+                          "Cheaper samples, and lower scores -- say so when reporting."),
+    ] = None,
+    trials: Annotated[
+        int,
+        typer.Option(help="Run each task this many times and report pass^k and the "
+                          "spread. One run cannot tell a real change from judge "
+                          "variance -- 260 of the 300 tasks are LLM-judged."),
+    ] = 1,
+    split: Annotated[
+        str,
+        typer.Option(help="'all', 'dev' (tune against these), or 'holdout' (never "
+                          "tuned against; the honest number). The split is a hash "
+                          "of the task id, so it does not move between runs."),
+    ] = "all",
+    lesson_bank: Annotated[
+        Optional[Path],
+        typer.Option(help="Lesson bank to read and write (default: ~/.otto/memory/"
+                          "lessons.db). Point a measurement at its own file so it "
+                          "does not learn from -- or teach -- your working one."),
+    ] = None,
+    no_learning: Annotated[
+        bool,
+        typer.Option("--no-learning", help="Read no lessons and write none. This is "
+                                           "the compute-matched baseline every "
+                                           "self-improvement claim has to be shown "
+                                           "beside."),
+    ] = False,
+    as_json: Annotated[bool, typer.Option("--json", help="Print the full report as JSON.")] = False,
+) -> None:
+    if architecture not in {"graph", "single"}:
+        err.print(f"architecture must be 'graph' or 'single', not {architecture!r}")
+        raise typer.Exit(2)
+    if split not in {"all", "dev", "holdout"}:
+        err.print(f"split must be 'all', 'dev' or 'holdout', not {split!r}")
+        raise typer.Exit(2)
+    if trials < 1:
+        err.print("trials must be at least 1")
+        raise typer.Exit(2)
+
+    try:
+        root = claw_root(claw_root_opt)
+        claw = load_claw(root)
+    except ClawEvalUnavailable as exc:
+        err.print(str(exc))
+        raise typer.Exit(2) from None
+
+    cfg = claw.load_config(str(config) if config else None)
+    # Their own factory, so the judge model, key and base URL come from the
+    # same config a `claw-eval run` would use rather than from a second copy.
+    judge = claw.cli._make_judge(cfg, SimpleNamespace(no_judge=no_judge, judge_model=None))
+
+    # Split BEFORE the limit, so --limit takes the first N of the chosen side
+    # rather than trimming the pool and then splitting a different set each
+    # time the pool changes.
+    tasks = select_tasks(root, tag=tag, pattern=task)
+    if split != "all":
+        development, reserved = split_tasks(tasks)
+        tasks = reserved if split == "holdout" else development
+    if limit:
+        tasks = tasks[:limit]
+    if not tasks:
+        err.print("no tasks matched")
+        raise typer.Exit(1)
+
+    out_dir = Path(trace_dir) if trace_dir else root / "traces" / f"otto-{architecture}-{time.strftime('%y%m%d-%H%M')}"
+    out_dir.mkdir(parents=True, exist_ok=True)
+    err.print(f"{len(tasks)} task(s) -> {out_dir}")
+
+    # What the loop is allowed to learn, decided once, here, rather than per
+    # task -- the discipline is a property of the whole measurement.
+    #
+    #   --no-learning   nothing read, nothing written. The baseline.
+    #   --split holdout lessons read, none written back. Tests TRANSFER: the
+    #                   loop never tunes against these tasks, which is the
+    #                   difference one evolved system got 31.7 points wrong.
+    #   otherwise       read and write. This is where lessons come from.
+    learning = ExitStack()
+    if no_learning:
+        learning.enter_context(bind_bank(None))
+        # Routing adapts from observed outcomes too, so the baseline has to
+        # hold that still as well. A "no learning" arm that quietly reordered
+        # the model chain partway through would be measuring two things.
+        learning.enter_context(bind_log(None))
+    else:
+        if lesson_bank:
+            learning.enter_context(bind_bank(MemoryStore(lesson_bank)))
+            # Beside the bank, so a measurement's routing evidence travels
+            # with its lessons instead of leaking into the working install.
+            learning.enter_context(bind_log(Path(lesson_bank).with_suffix(".seats.db")))
+        if split == "holdout":
+            # Read what the development runs learned, write nothing back --
+            # lessons and routing evidence alike. That is what makes the
+            # held-out number answer "does this TRANSFER" rather than "did the
+            # loop find something that works on what it was tuned on".
+            learning.enter_context(read_only())
+            learning.enter_context(routing_read_only())
+    err.print(
+        "learning: " + ("off (baseline)" if no_learning else
+                        "read-only (held out)" if split == "holdout" else "on")
+    )
+
+    outcomes = []
+    try:
+        _run_tasks(claw, tasks, outcomes, out_dir=out_dir, cfg=cfg, judge=judge,
+                   architecture=architecture, port_offset=port_offset,
+                   max_seconds=max_seconds, trials=trials)
+    finally:
+        learning.close()
+
+    report = _build_report(
+        outcomes, claw, architecture=architecture, tag=tag, split=split,
+        trials=trials, max_seconds=max_seconds, out_dir=out_dir, cfg=cfg,
+        judge=judge, no_learning=no_learning,
+    )
+    (out_dir / "otto_summary.json").write_text(json.dumps(report, indent=2))
+
+    if as_json:
+        out.print_json(data=report)
+        return
+    _print_summary(report, out_dir, split)
+
+
+def _run_tasks(claw, tasks, outcomes, *, out_dir, cfg, judge, architecture,
+               port_offset, max_seconds, trials) -> None:
+    for i, task_yaml in enumerate(tasks, 1):
+        name = task_yaml.parent.name
+        err.print(f"[{i}/{len(tasks)}] {name}")
+        missing = missing_service_keys(claw.TaskDefinition.from_yaml(task_yaml))
+        if missing:
+            err.print(
+                f"  [warn]{', '.join(missing)} not set[/] -- this task's service "
+                "reaches the real internet and will return nothing, so the score "
+                "below measures the environment, not the agent"
+            )
+        try:
+            outcome = run_task_file(
+                claw, task_yaml,
+                trace_dir=out_dir, cfg=cfg, judge=judge,
+                architecture=architecture, port_offset=port_offset,
+                max_seconds=max_seconds, trials=trials,
+            )
+        except Exception as exc:
+            # One task's container or service failing is not a reason to lose
+            # the other 299 -- report it and carry on, the way their batch does.
+            err.print(f"  [bad]harness error[/] {type(exc).__name__}: {exc}")
+            continue
+        outcomes.append(outcome)
+        flag = "[ok]pass[/]" if outcome.passed else "[warn]fail[/]"
+        for item in outcome.checklist:
+            err.print(f"    [{item.get('status', '?')}] {item.get('text', '')[:90]}")
+        detail = f" ({outcome.error})" if outcome.error else ""
+        spread = (
+            " trials=" + "/".join(f"{t:.2f}" for t in outcome.trials)
+            if len(outcome.trials) > 1 else ""
+        )
+        out.print(
+            f"  {flag} score={outcome.task_score:.2f} "
+            f"completion={outcome.completion:.2f} service-tools={outcome.tool_calls} "
+            f"actions={outcome.agent_actions} calls={outcome.model_calls} "
+            f"{outcome.wall_time_s:.0f}s{spread}{detail}"
+        )
+
+
+def _build_report(outcomes, claw, *, architecture, tag, split, trials,
+                  max_seconds, out_dir, cfg, judge, no_learning) -> dict:
+    return {
+        "architecture": architecture,
+        "learning": "off" if no_learning else "read-only" if split == "holdout" else "on",
+        "tag": tag,
+        "max_seconds": max_seconds,
+        "split": split,
+        "trials": trials,
+        "trace_dir": str(out_dir),
+        # What the scores mean. Two reports whose fingerprints differ are not
+        # comparable, however similar the numbers look.
+        "grading": grading_fingerprint(claw, cfg, judge),
+        "summary": _summary(outcomes, claw),
+        "tasks": [
+            {
+                "task_id": o.task_id, "task_score": o.task_score, "passed": o.passed,
+                "completion": o.completion, "robustness": o.robustness,
+                "communication": o.communication, "safety": o.safety,
+                "tool_calls": o.tool_calls, "agent_actions": o.agent_actions,
+                "checklist": o.checklist, "model_calls": o.model_calls,
+                "trials": o.trials,
+                "wall_time_s": o.wall_time_s, "error": o.error,
+            }
+            for o in outcomes
+        ],
+    }
+
+
+def _print_summary(report: dict, out_dir: Path, split: str) -> None:
+    s = report["summary"]
+
+    # Said BEFORE the numbers, not after. A caveat printed underneath a mean
+    # score is read after the score has already been believed, and the whole
+    # failure this guards against is a single run being quoted as a result.
+    if s.get("tasks") and not s.get("is_evidence", True):
+        trials = s.get("trials", 1)
+        err.print(
+            f"\n[bad]NOT EVIDENCE: {trials} trial(s) per task.[/] "
+            f"[warn]Identical code and configuration have scored "
+            f"{OBSERVED_SINGLE_RUN_SPREAD:.2f} apart on a single task here, so "
+            f"any before/after difference below roughly 0.3 in what follows is "
+            f"noise. Do not quote these numbers as a result.[/]\n"
+            f"[muted]Re-run with --trials {MIN_TRIALS_FOR_EVIDENCE} and read "
+            f"pass^k, not the mean.[/]"
+        )
+
+    out.print(
+        f"\n{s.get('passed', 0)}/{s.get('tasks', 0)} passed "
+        f"({s.get('pass_rate', 0):.1%})  "
+        f"{'mean score' if s.get('is_evidence', True) else 'score (1 run)'} "
+        f"{s.get('mean_task_score', 0):.3f}  "
+        f"completion {s.get('mean_completion', 0):.3f}  "
+        f"{s.get('errors', 0)} harness/agent error(s)"
+    )
+    out.print(
+        f"cost: {s.get('mean_model_calls', 0):.1f} model calls and "
+        f"{s.get('mean_wall_time_s', 0):.0f}s per task"
+    )
+    if "mean_pass_hat_k" in s:
+        k = s["trials"]
+        out.print(
+            f"reliability over {k} trials: pass^{k} {s['mean_pass_hat_k']:.3f}  "
+            f"pass@{k} {s['mean_pass_at_k']:.3f}  "
+            f"mean spread {s['score_spread']:.3f}"
+        )
+    kinds = (s.get("failures") or {}).get("failures_by_kind") or {}
+    if kinds:
+        total_failed = s.get("tasks", 0) - s.get("passed", 0)
+        out.print(
+            "failures by kind: "
+            + "  ".join(f"{kind} {count}" for kind, count in kinds.items())
+            + f"  (of {total_failed} failed)"
+        )
+    cost = s.get("tool_cost") or {}
+    if cost.get("calls"):
+        # Per SEAT is the half that answers the question: Otto routes across
+        # four vendors and several capability tiers, and model strength is
+        # what decides whether an agent manages a seventeen-tool menu at all.
+        busiest = "  ".join(f"{tool} {n}" for tool, n in
+                            list(cost["by_tool"].items())[:5])
+        out.print(
+            f"tools: {cost['calls']} calls, {cost['failures']} failed  |  "
+            f"busiest: {busiest}"
+        )
+        out.print(
+            "per seat: "
+            + "  ".join(f"{seat} {n}" for seat, n in cost["by_seat"].items())
+        )
+
+    grading = report["grading"]
+    out.print(
+        f"grading: {grading['otto_grading_path']} / claw {grading['claw_eval_revision'] or '?'} "
+        f"/ judge {grading['judge']} / split {split}"
+    )
+    out.print(f"report: {out_dir / 'otto_summary.json'}")

agent/cli/eval_compaction.py

@@ -0,0 +1,106 @@
+"""`otto eval-compaction` -- what each compaction policy actually loses.
+
+The instrument Otto did not have. `otto eval-memory` scores whether raw
+evidence stays RETRIEVABLE, which the chunk store nearly guarantees; this
+scores whether the constraint is still in what the prompt reads. Those are
+different questions and only the second one distinguishes compaction policies.
+
+Offline and free by default: the summariser is a deterministic stand-in, so a
+policy comparison measures the POLICY rather than whichever model happened to
+summarise that day.
+"""
+from __future__ import annotations
+
+import json
+import tempfile
+from pathlib import Path
+from typing import Optional
+
+import typer
+from rich import box
+from rich.table import Table
+from typing_extensions import Annotated
+
+from agent.cli.ui import err, out
+from agent.eval.compaction_bench import POLICIES, run_matrix
+
+
+def eval_compaction_cmd(
+    turns: Annotated[
+        int, typer.Option(help="How long a conversation to replay. Longer means "
+                               "more compaction rounds, which is where policies diverge."),
+    ] = 120,
+    policy: Annotated[
+        Optional[str],
+        typer.Option(help="Run one policy instead of the whole matrix."),
+    ] = None,
+    live: Annotated[
+        bool,
+        typer.Option("--live", help="Use the real Task.SUMMARIZE model instead of the "
+                                    "deterministic stand-in. Answers a question about "
+                                    "the MODEL, not about the policy, and costs calls."),
+    ] = False,
+    out_dir: Annotated[
+        Optional[Path], typer.Option(help="Where to put the stores (default: a temp dir)."),
+    ] = None,
+    as_json: Annotated[bool, typer.Option("--json", help="Print the report as JSON.")] = False,
+) -> None:
+    chosen = POLICIES
+    if policy:
+        if policy not in POLICIES:
+            err.print(f"no policy {policy!r} -- have {', '.join(POLICIES)}")
+            raise typer.Exit(2)
+        chosen = {policy: POLICIES[policy]}
+
+    summarize = None
+    if live:
+        from agent.memory.wiring import summarize_for_memory
+
+        summarize = summarize_for_memory
+        err.print("[warn]--live[/] spends model calls, one per compaction round per policy")
+
+    root = Path(out_dir) if out_dir else Path(tempfile.mkdtemp(prefix="otto-compaction-"))
+    root.mkdir(parents=True, exist_ok=True)
+    err.print(f"{len(chosen)} policy/policies over {turns} turns -> {root}")
+
+    results = run_matrix(root, turns=turns, policies=chosen, summarize=summarize)
+
+    if as_json:
+        out.print_json(data={
+            "turns": turns, "live": live,
+            "policies": [
+                {
+                    "name": r.name, "survives": r.survives, "recoverable": r.recoverable,
+                    "total": r.total, "survival_rate": round(r.survival_rate, 3),
+                    "recovery_rate": round(r.recovery_rate, 3),
+                    "view_chars": r.view_chars, "compactions": r.compactions,
+                    "missing": r.missing,
+                }
+                for r in results
+            ],
+        })
+        return
+
+    table = Table(box=box.SIMPLE, pad_edge=False)
+    table.add_column("policy")
+    table.add_column("in the view", justify="right")
+    table.add_column("recoverable", justify="right")
+    table.add_column("view chars", justify="right")
+    table.add_column("rounds", justify="right")
+    for r in results:
+        # The number that matters is the first: what the model will actually
+        # read without being told to go looking.
+        style = "ok" if r.survival_rate >= 0.75 else "warn" if r.survival_rate else "bad"
+        table.add_row(
+            r.name,
+            f"[{style}]{r.survives}/{r.total}[/]",
+            f"{r.recoverable}/{r.total}",
+            f"{r.view_chars:,}",
+            str(r.compactions),
+        )
+    out.print(table)
+    out.print(
+        "'in the view' is what a prompt is built from; 'recoverable' is what "
+        "recall_memory could still find. The gap between them is text the "
+        "model will not see unless it thinks to go looking."
+    )