otto-cli-agent 0.1.0__py3-none-any.whl

agent/cli/eval_hle.py

@@ -0,0 +1,92 @@
+"""`otto eval-hle` -- Humanity's Last Exam against Otto (agent/eval/hle_bench.py).
+
+Two modes on the same sample, because the interesting number is not HLE's
+published figure for this model but what Otto's graph does to it: `raw` asks
+the model once, `agent` runs the whole router/planner/solver/evaluator graph
+over the identical question and reports how many model calls that cost.
+
+The dataset is gated -- accept the terms at huggingface.co/datasets/cais/hle
+and set HF_TOKEN. Start with a small --limit: `agent` mode spends a full graph
+run per question.
+"""
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Optional
+
+import typer
+from typing_extensions import Annotated
+
+from agent.cli.ui import err, out
+from agent.eval.hle_bench import (
+    DEFAULT_CACHE,
+    DatasetGated,
+    download_hle,
+    load_hle,
+    run_hle,
+    sample_questions,
+    summarise,
+)
+
+
+def eval_hle_cmd(
+    mode: Annotated[
+        str, typer.Option(help="'raw' (one model call per question) or 'agent' (the whole graph)."),
+    ] = "raw",
+    limit: Annotated[
+        Optional[int], typer.Option(help="Score only this many questions (sampled deterministically)."),
+    ] = 25,
+    seed: Annotated[int, typer.Option(help="Sampling seed, so two modes score the SAME questions.")] = 0,
+    data_path: Annotated[
+        Optional[Path], typer.Option(help=f"Cached HLE parquet (default: {DEFAULT_CACHE}).")
+    ] = None,
+    hf_token: Annotated[
+        Optional[str], typer.Option(help="Hugging Face token; defaults to $HF_TOKEN.")
+    ] = None,
+    show_items: Annotated[
+        bool, typer.Option("--show-items", help="Print every question, answer and verdict."),
+    ] = False,
+    as_json: Annotated[bool, typer.Option("--json", help="Print the full report as JSON.")] = False,
+) -> None:
+    if mode not in {"raw", "agent"}:
+        err.print(f"mode must be 'raw' or 'agent', not {mode!r}")
+        raise typer.Exit(2)
+
+    path = data_path or DEFAULT_CACHE
+    try:
+        download_hle(path, token=hf_token)
+    except DatasetGated as exc:
+        err.print(str(exc))
+        raise typer.Exit(1)
+
+    rows, skipped = sample_questions(load_hle(path), limit=limit, seed=seed)
+    out.print(f"scoring {len(rows)} text-only question(s) in {mode} mode "
+        f"({skipped} image question(s) skipped -- Otto's chat path is text-only)")
+
+    def _progress(r):
+        mark = "correct" if r.correct else "wrong  "
+        out.print(f"  {mark}  {r.llm_calls:>3} call(s)  {r.question[:72]}")
+
+    results = run_hle(rows, mode=mode, on_result=None if as_json else _progress)
+    report = summarise(results)
+
+    if as_json:
+        out.print(json.dumps({"mode": mode, "skipped_image_questions": skipped,
+                        "summary": report,
+                        "items": [r.__dict__ for r in results]}, indent=2))
+        return
+
+    out.print("")
+    out.print(f"accuracy: {report['accuracy']:.1%} of {report['n']}"
+        f"   model calls: {report['llm_calls_total']} "
+        f"({report['llm_calls_per_question']:.1f} per question)")
+    for name, stats in report["by_category"].items():
+        out.print(f"  {name:<34} {stats['accuracy']:>6.1%}  (n={stats['n']})")
+
+    if show_items:
+        out.print("")
+        for r in results:
+            out.print(f"[{'correct' if r.correct else 'wrong'}] {r.question[:110]}")
+            out.print(f"   expected: {r.correct_answer[:110]}")
+            out.print(f"   extracted: {r.extracted[:110]}")

agent/cli/eval_memory.py

@@ -0,0 +1,253 @@
+"""`otto eval-memory`: run agent/eval/memory_bench.py's LoCoMo-based memory
+benchmark and report coverage. See that module's own docstring for what's
+actually measured (store/visible-verbatim/recall/answerable coverage per QA
+category, plus a compression ratio) and why the two `--x-budget`/`--y-budget`
+options exist: real LoCoMo conversations are small enough that Otto's real
+production budget (agent/memory/queue.py's X_BUDGET/Y_BUDGET) never triggers
+compaction at all, so a run at the defaults mostly reports "everything is
+still verbatim, nothing needed compacting" -- correct, but it doesn't
+exercise the compaction+recall code path. Passing smaller values here forces
+that path to actually run, at the cost of no longer matching Otto's real
+per-turn budget.
+
+No Langfuse tracking here (unlike `otto eval`, agent/cli/eval.py) -- this
+benchmark doesn't run the pipeline or the router at all in its default,
+offline mode (`--live` opts into one real LLM call per compaction, via
+agent.memory.wiring.summarize_for_memory, agent/eval/memory_bench.py's own
+module docstring), so there's no per-item trace worth syncing there.
+
+`--show-items failures|all` prints each scored QA item's evidence text next
+to recall()'s raw output -- worth reaching for whenever a coverage number
+looks suspicious, since `recalled` is an exact-substring match (module
+docstring point 3), not a fuzzy one: reading the two side by side is how
+you tell a real semantic-search find from a coincidental match. It also
+surfaces a real gap in the OFFLINE (`--no-live`, default) summarizer:
+_canned_summarize groups raw turns into fixed-size blocks with placeholder
+bullet text, and recall() returns a matched bullet's ENTIRE underlying raw
+text -- so offline, "recalled" mostly tests "did the right block rank in
+top-k," not "did it find the right sentence." `--live`'s real, topic-
+scoped bullet summaries are the stricter version of this same test.
+"""
+from __future__ import annotations
+
+import json as json_module
+from pathlib import Path
+from typing import Annotated, Optional
+
+import typer
+from rich import box
+from rich.table import Table
+
+from agent.cli.ui import err, out
+from agent.memory.retrieval import (
+    DEFAULT_MAX_CHUNKS,
+    DEFAULT_NEIGHBOUR_WINDOW,
+    DEFAULT_TOKEN_BUDGET,
+)
+from agent.eval.memory_bench import (
+    BenchmarkReport,
+    DEFAULT_CACHE,
+    download_locomo,
+    load_locomo,
+    recalled_text_has_unparsed_bullet,
+    run_benchmark,
+)
+
+
+def eval_memory_cmd(
+    live: Annotated[
+        bool,
+        typer.Option(
+            "--live/--no-live",
+            help="Use the real Task.SUMMARIZE model + local embeddings (needs INCEPTION_API_KEY) "
+                 "instead of the deterministic offline canned summarizer.",
+        ),
+    ] = False,
+    samples: Annotated[
+        Optional[int], typer.Option(help="Only run the first N of LoCoMo's 10 conversations.")
+    ] = None,
+    max_turns: Annotated[
+        Optional[int], typer.Option(help="Truncate each conversation to its first N turns (fast smoke run).")
+    ] = None,
+    x_budget: Annotated[
+        Optional[int],
+        typer.Option(help="TieredQueue's X budget (tokens). Defaults to the "
+                          "PRODUCTION budget, which LoCoMo's shorter "
+                          "conversations do not overflow -- so the default "
+                          "compacts nothing and the run is refused. Try 600."),
+    ] = None,
+    y_budget: Annotated[
+        Optional[int],
+        typer.Option(help="TieredQueue's Y budget (tokens). Same as --x-budget: "
+                          "the production default never binds here. Try 1500."),
+    ] = None,
+    top_k: Annotated[int, typer.Option(help="How many bullets recall() considers per question (stage 1).")] = 5,
+    max_chunks: Annotated[
+        int,
+        typer.Option(help="How many raw chunks recall() actually returns (stage 2's cap)."),
+    ] = DEFAULT_MAX_CHUNKS,
+    neighbours: Annotated[
+        int,
+        typer.Option(help="How many chunks either side of each returned chunk to include."),
+    ] = DEFAULT_NEIGHBOUR_WINDOW,
+    token_budget: Annotated[
+        int,
+        typer.Option(help="Hard ceiling (tokens) on how much text one recall() returns."),
+    ] = DEFAULT_TOKEN_BUDGET,
+    data_path: Annotated[
+        Optional[Path], typer.Option(help="Path to a cached locomo10.json (default: agent/eval/data/locomo10.json).")
+    ] = None,
+    force_download: Annotated[
+        bool, typer.Option("--force-download", help="Re-download locomo10.json even if a cached copy exists.")
+    ] = False,
+    json: Annotated[
+        bool, typer.Option("--json", help="Print the full report as JSON instead of a table.")
+    ] = False,
+    show_items: Annotated[
+        str,
+        typer.Option(
+            help="Print per-question detail after the table: 'none' (default), 'failures' "
+                 "(answerable=False items only -- the interesting/debug case), or 'all'. Each "
+                 "item shows its evidence text and recall()'s raw output side by side, since "
+                 "`recalled` is an exact-substring match, not a fuzzy one -- useful for checking "
+                 "whether a pass/miss is real or an artifact of the conversation repeating similar "
+                 "phrasing. Ignored when --json is set (the JSON report already includes both).",
+        ),
+    ] = "none",
+) -> None:
+    """Score agent/memory/'s tiered queue against the LoCoMo long-conversation-memory dataset."""
+    if show_items not in ("none", "failures", "all"):
+        err.print("[bad]--show-items must be 'none', 'failures', or 'all'[/]")
+        raise typer.Exit(2)
+
+    path = data_path or DEFAULT_CACHE
+    with err.status(f"fetching LoCoMo dataset ({path})…"):
+        try:
+            download_locomo(path, force=force_download)
+        except Exception as exc:
+            err.print(f"[bad]could not download the LoCoMo dataset: {exc}[/]")
+            err.print("[muted]pass --data-path to point at an already-downloaded locomo10.json[/]")
+            raise typer.Exit(1) from exc
+        all_samples = load_locomo(path)
+
+    if samples is not None:
+        all_samples = all_samples[:samples]
+    if not all_samples:
+        err.print("[bad]no LoCoMo samples to run[/]")
+        raise typer.Exit(1)
+
+    with err.status(f"running {len(all_samples)} conversation(s), live={live}…"):
+        report = run_benchmark(
+            all_samples, live=live, top_k=top_k, max_chunks=max_chunks,
+            neighbour_window=neighbours, token_budget=token_budget, max_turns=max_turns,
+            x_budget=x_budget, y_budget=y_budget,
+        )
+
+    if json:
+        out.print(json_module.dumps(report.to_dict(), indent=2))
+        return
+
+    summary = report.summary()
+
+    # REFUSE, rather than print a perfect-looking empty result. A run at the
+    # production budgets never compacts LoCoMo's shorter conversations, so
+    # every citation stays in the verbatim window, retrieval is never asked a
+    # question, and the table comes out `recalled 0%` beside `answerable
+    # 100%` -- which reads as a pass. The numbers are not wrong so much as
+    # measured off nothing, and printing them at all is what let this happen
+    # unnoticed. See memory_bench.NO_COMPACTION_RATIO.
+    if summary["conversation_count"] and not summary["recall_exercised"]:
+        ratio = summary["overall_compression_ratio"]
+        err.print(
+            "[bad]nothing was compacted at these budgets, so recall was never "
+            "exercised -- this run measured nothing and no coverage table is "
+            "printed for it.[/]\n"
+            f"[muted]final_view/raw token ratio "
+            f"{'-' if ratio is None else f'{ratio:.3f}'} over "
+            f"{summary['conversation_count']} conversation(s), at "
+            f"--x-budget {x_budget} --y-budget {y_budget}.[/]\n"
+            "[warn]re-run with smaller --x-budget/--y-budget -- e.g. "
+            "--x-budget 600 --y-budget 1500 -- so the conversations overflow "
+            "and the retrieval path actually runs.[/]"
+        )
+        raise typer.Exit(1)
+
+    t = Table(box=box.SIMPLE, header_style="muted")
+    t.add_column("category", style="spec")
+    t.add_column("n", justify="right")
+    t.add_column("store", justify="right")
+    t.add_column("verbatim", justify="right")
+    t.add_column("recalled", justify="right")
+    t.add_column("answerable", justify="right")
+
+    def _pct(value: Optional[float]) -> str:
+        return "-" if value is None else f"{value * 100:.0f}%"
+
+    for name, rates in summary["by_category"].items():
+        t.add_row(
+            name, str(rates["n"]), _pct(rates["store_coverage"]),
+            _pct(rates["visible_verbatim_coverage"]), _pct(rates["recall_coverage"]),
+            _pct(rates["answerable_coverage"]),
+        )
+    overall = summary["overall"]
+    t.add_row(
+        "[chosen]overall[/]", str(overall["n"]), _pct(overall["store_coverage"]),
+        _pct(overall["visible_verbatim_coverage"]), _pct(overall["recall_coverage"]),
+        _pct(overall["answerable_coverage"]),
+    )
+    out.print(t)
+
+    ratio = summary["overall_compression_ratio"]
+    ratio_text = "-" if ratio is None else f"{ratio:.3f}"
+    out.print(
+        f"[muted]{summary['conversation_count']} conversation(s) -- "
+        f"final_view/raw token ratio: {ratio_text}[/]"
+    )
+
+    unparsed = summary["unparsed_bullet_count"]
+    if unparsed:
+        err.print(
+            f"[warn]{unparsed}/{summary['bullet_count']} stored bullets are unparsed-summary "
+            f"placeholders (the summarizer's reply didn't parse for that compaction) -- "
+            f"recall_coverage is partly measuring ranking against content-free text, not a "
+            f"real summary. --show-items all will show which questions land on one.[/]"
+        )
+
+    if show_items != "none":
+        _print_items(report, show_items)
+
+    if overall["n"] and overall["store_coverage"] is not None and overall["store_coverage"] < 1.0:
+        err.print("[bad]store_coverage < 100% -- the engine lost cited evidence; this is a real bug[/]")
+        raise typer.Exit(1)
+
+
+def _snippet(text: str, limit: int = 280) -> str:
+    text = " ".join(text.split())
+    return text if len(text) <= limit else text[:limit] + "…"
+
+
+def _print_items(report: BenchmarkReport, which: str) -> None:
+    """Per-question detail: for every scored QA item matching `which`, the
+    exact evidence text next to recall()'s raw output -- the pair
+    `recalled` was computed from (an exact substring check, module
+    docstring point 3), so reading them side by side is how you tell a
+    real find from a coincidental match in a conversation that repeats
+    similar phrasing often.
+    """
+    out.print("")
+    shown = 0
+    for conv in report.conversations:
+        for r in conv.scored_results():
+            if which == "failures" and r.answerable:
+                continue
+            shown += 1
+            verdict = "[ok]answerable[/]" if r.answerable else "[bad]MISS[/]"
+            via = "verbatim" if r.visible_verbatim else ("recalled" if r.recalled else "neither")
+            flag = " [warn]unparsed-bullet[/]" if recalled_text_has_unparsed_bullet(r.recalled_text) else ""
+            out.print(f"[spec]{conv.sample_id}[/] [muted]({via})[/] {verdict}{flag}")
+            out.print(f"  Q: {r.question}")
+            out.print(f"  evidence: {_snippet(r.evidence_text)}")
+            out.print(f"  recalled: {_snippet(r.recalled_text)}")
+            out.print("")
+    if shown == 0:
+        out.print("[muted](nothing matched --show-items filter)[/]")

agent/cli/eval_swe.py

@@ -0,0 +1,172 @@
+"""`otto eval-swe` -- SWE-bench Verified, graded by each repository's own tests.
+
+The only benchmark here whose verdict nobody involved can argue with: 500 real
+GitHub issues, and the maintainers' own test suite decides. See
+agent/eval/swe_bench.py for what Otto is and is not told.
+
+Needs Docker. The official per-instance images are about 3.5GB each and are
+pulled on first use, so start with --limit 1 and expect the first run of a
+repository to spend several minutes downloading before the agent starts.
+"""
+from __future__ import annotations
+
+import json
+import time
+from pathlib import Path
+from typing import Optional
+
+import typer
+from typing_extensions import Annotated
+
+from agent.cli.ui import err, out
+from agent.eval.swe_bench import (
+    SweBenchUnavailable, load_dataset, resolve_image, run_instance, select,
+)
+
+
+def eval_swe_cmd(
+    repo: Annotated[
+        Optional[str], typer.Option(help="Only instances from repositories matching this."),
+    ] = None,
+    instance: Annotated[
+        Optional[str], typer.Option(help="Substring of an instance id, e.g. 'astropy-12907'."),
+    ] = None,
+    difficulty: Annotated[
+        Optional[str],
+        typer.Option(help="Filter on the dataset's own estimate: '<15', '15 min', '1-4', '>4'."),
+    ] = None,
+    limit: Annotated[int, typer.Option(help="Run at most this many instances.")] = 1,
+    max_seconds: Annotated[
+        float, typer.Option(help="Wall-clock budget per instance, before grading."),
+    ] = 1800.0,
+    max_model_calls: Annotated[
+        Optional[int], typer.Option(help="Override the default spend ceiling per instance."),
+    ] = None,
+    trace_dir: Annotated[
+        Optional[Path], typer.Option(help="Where to write each instance's diff."),
+    ] = None,
+    dataset: Annotated[
+        Optional[Path], typer.Option(help="Cached parquet (default: agent/eval/data/)."),
+    ] = None,
+    as_json: Annotated[bool, typer.Option("--json", help="Print the report as JSON.")] = False,
+) -> None:
+    try:
+        instances = select(
+            load_dataset(dataset), repo=repo, pattern=instance,
+            difficulty=difficulty, limit=limit,
+        )
+    except SweBenchUnavailable as exc:
+        err.print(str(exc))
+        raise typer.Exit(2) from None
+
+    if not instances:
+        err.print("no instances matched")
+        raise typer.Exit(1)
+
+    out_dir = Path(trace_dir) if trace_dir else Path("/tmp") / f"otto-swe-{time.strftime('%y%m%d-%H%M')}"
+    err.print(f"{len(instances)} instance(s) -> {out_dir}")
+
+    outcomes = []
+    for i, item in enumerate(instances, 1):
+        err.print(f"[{i}/{len(instances)}] {item.instance_id}  ({item.difficulty or 'unrated'})")
+        image, emulated = resolve_image(item.instance_id)
+        err.print(f"  image {image}"
+                  + ("  [warn](x86_64 under emulation -- no arm64 build)[/]"
+                     if emulated else ""))
+        try:
+            outcome = run_instance(
+                item, max_seconds=max_seconds,
+                max_model_calls=max_model_calls, trace_dir=out_dir,
+            )
+        except SweBenchUnavailable as exc:
+            err.print(f"  [bad]harness error[/] {exc}")
+            continue
+        outcomes.append(outcome)
+        flag = "[ok]resolved[/]" if outcome.resolved else "[warn]not resolved[/]"
+        detail = f" ({outcome.error})" if outcome.error else ""
+        out.print(
+            f"  {flag}  {outcome.grade.summary()}  "
+            f"{outcome.diff_lines} changed line(s)  calls={outcome.model_calls}  "
+            f"{outcome.wall_time_s:.0f}s{detail}"
+        )
+        if outcome.grade.error:
+            err.print(f"    [bad]{outcome.grade.error}[/]")
+        for failure in outcome.grade.failures[:3]:
+            err.print(f"    still failing: {failure[:100]}")
+
+    if not outcomes:
+        raise typer.Exit(1)
+
+    resolved = sum(1 for o in outcomes if o.resolved)
+    # How much of this number came from emulated instances, and how it splits.
+    #
+    # Upstream publishes arm64 for only part of the set, and django -- 231 of
+    # the 500 instances -- had no arm64 build in every instance tried. So on
+    # Apple silicon roughly half the benchmark runs under emulation, several
+    # times slower, and a resolve rate that mixes the two is not a sample of
+    # SWE-bench. swe_bench.EMULATION_TIME_FACTOR stops the budget being the
+    # thing that decides those instances; it does NOT make them comparable,
+    # which is why the split is reported rather than smoothed away.
+    emulated = [o for o in outcomes if o.emulated]
+    native = [o for o in outcomes if not o.emulated]
+
+    def _rate(group):
+        if not group:
+            return None
+        return round(sum(1 for o in group if o.resolved) / len(group), 4)
+
+    report = {
+        "instances": len(outcomes),
+        "resolved": resolved,
+        "resolve_rate": round(resolved / len(outcomes), 4),
+        "emulated_instances": len(emulated),
+        "native_instances": len(native),
+        "emulated_resolve_rate": _rate(emulated),
+        "native_resolve_rate": _rate(native),
+        #: The repositories whose instances ran emulated, which is the thing
+        #: a reader needs to judge whether the sample is representative --
+        #: "half of it was django, emulated" is a different number from the
+        #: same rate measured natively.
+        "emulated_repos": sorted({o.instance_id.split("__")[0] for o in emulated}),
+        "mean_model_calls": round(sum(o.model_calls for o in outcomes) / len(outcomes), 1),
+        "mean_wall_time_s": round(sum(o.wall_time_s for o in outcomes) / len(outcomes), 1),
+        "trace_dir": str(out_dir),
+        "results": [
+            {
+                "instance_id": o.instance_id, "resolved": o.resolved,
+                "fail_to_pass": [o.grade.fail_to_pass_passed, o.grade.fail_to_pass_total],
+                "pass_to_pass": [o.grade.pass_to_pass_passed, o.grade.pass_to_pass_total],
+                "diff_lines": o.diff_lines, "model_calls": o.model_calls,
+                "emulated": o.emulated,
+                "wall_time_s": round(o.wall_time_s, 1),
+                "error": o.error or o.grade.error,
+            }
+            for o in outcomes
+        ],
+    }
+    out_dir.mkdir(parents=True, exist_ok=True)
+    (out_dir / "report.json").write_text(json.dumps(report, indent=2))
+
+    if as_json:
+        out.print_json(data=report)
+        return
+    out.print(
+        f"\n{resolved}/{len(outcomes)} resolved ({report['resolve_rate']:.1%})  "
+        f"{report['mean_model_calls']:.0f} model calls and "
+        f"{report['mean_wall_time_s']:.0f}s per instance"
+    )
+    if emulated:
+        # A SWE-bench number from a machine that emulated part of its sample
+        # has to say so, in the same breath as the number.
+        native_text = (
+            f"{report['native_resolve_rate']:.1%}" if native else "no native instances"
+        )
+        err.print(
+            f"[warn]{len(emulated)}/{len(outcomes)} instances ran under x86_64 "
+            f"emulation (no arm64 build): "
+            f"{', '.join(report['emulated_repos'])}.[/]\n"
+            f"[muted]emulated {report['emulated_resolve_rate']:.1%} vs native "
+            f"{native_text} -- quote the split, not the combined rate, and say "
+            f"this machine emulated part of the sample.[/]"
+        )
+    out.print(f"report: {out_dir / 'report.json'}")

agent/cli/lessons.py

@@ -0,0 +1,97 @@
+"""`otto lessons` -- read, move, and if need be empty, what Otto has learned.
+
+A self-improving system whose learned state cannot be read is not a system
+anyone should trust, and this is the cheapest possible remedy: the bank is a
+few short lines of text, so print them.
+
+There is a second reason it exists. Evolution that is not working is a real
+and documented outcome -- across five methods and three frontier models the
+measured gains were around +1%, and negative in every regime for the
+strongest model. Being able to look at the bank and say "these lessons are
+rubbish" is how that gets noticed, and `--clear` is how it gets undone.
+
+`--export` / `--import` (2026-09-12) move a bank between machines as JSON,
+through the same duplicate adjudication a run's own lessons face. The TUI's
+"Export lessons…" / "Import lessons…" palette entries call the same two
+functions (agent/memory/lessons.py).
+"""
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Optional
+
+import typer
+from rich import box
+from rich.table import Table
+from rich.text import Text
+from typing_extensions import Annotated
+
+from agent.cli.ui import err, out
+from agent.memory.lessons import (
+    Lesson, all_lessons, bank_path, bind_bank, clear_bank, export_lessons, import_lessons,
+)
+from agent.memory.store import MemoryStore
+
+
+def lessons_table(lessons: list[Lesson], path: Path | None = None) -> Table:
+    """The bank as a table. Primitive Rich styles only, so it renders the same
+    in the REPL (whose THEME maps ok/warn to exactly these) and inside a
+    Textual Static, which never sees that THEME."""
+    title = f"{len(lessons)} lesson(s)" + (f" in {path}" if path else "")
+    t = Table(box=box.SIMPLE, header_style="dim", title=title, title_justify="left")
+    t.add_column("outcome")
+    t.add_column("when")
+    t.add_column("do", overflow="fold")
+    for lesson in lessons:
+        mark = Text("worked", style="bold green") if lesson.outcome == "worked" else Text("failed", style="yellow")
+        t.add_row(mark, lesson.cue, lesson.action)
+    return t
+
+
+def lessons_cmd(
+    bank: Annotated[
+        Optional[Path],
+        typer.Option(help="Which bank to read (default: ~/.otto/memory/lessons.db)."),
+    ] = None,
+    clear: Annotated[
+        bool,
+        typer.Option("--clear", help="Delete every lesson. Not undoable."),
+    ] = False,
+    export: Annotated[
+        Optional[Path],
+        typer.Option("--export", help="Write the bank to this JSON file."),
+    ] = None,
+    import_from: Annotated[
+        Optional[Path],
+        typer.Option("--import", help="Merge lessons from this JSON file into the bank."),
+    ] = None,
+    replace: Annotated[
+        bool,
+        typer.Option("--replace", help="With --import: empty the bank first."),
+    ] = False,
+) -> None:
+    path = bank or bank_path()
+    if import_from is None and not path.exists():
+        out.print(f"no lesson bank at {path} -- nothing learned yet")
+        return
+
+    store = MemoryStore(path)
+    with bind_bank(store):
+        if clear:
+            removed = clear_bank()
+            err.print(f"cleared {removed} lesson(s) from {path}")
+            return
+        if export is not None:
+            n = export_lessons(export)
+            err.print(f"exported {n} lesson(s) to {export}")
+            return
+        if import_from is not None:
+            report = import_lessons(import_from, replace=replace)
+            err.print(f"imported from {import_from}: {report.summary()}")
+            return
+
+        learned = all_lessons()
+        if not learned:
+            out.print(f"{path} is empty")
+            return
+        out.print(lessons_table(learned, path))

agent/cli/main.py

@@ -0,0 +1,67 @@
+from dotenv import load_dotenv
+from typing import Annotated
+import typer
+
+# Loaded here, at import time, before anything below it is imported -- not
+# inside bootstrap(). agent.cli.chat now pulls in the whole pipeline
+# (agent.pipeline.nodes), which constructs a module-level Router() the
+# moment it's imported, which reads *_API_KEY from the environment
+# immediately. bootstrap() only runs once Typer has already finished
+# importing every command module, which is too late for that first
+# Router() call -- it would always see an environment with no keys in it,
+# whatever's actually in .env.
+from agent.config.envfile import ENV_PATH  # the one file the setup screen writes
+
+load_dotenv(ENV_PATH)
+
+# Also before anything below: ~/.otto/routes.json may name custom endpoints
+# and per-task pins, and the module-level Router() in agent.pipeline.nodes
+# snapshots which providers exist the moment it is imported.
+from agent.router.overrides import apply_at_startup  # noqa: E402
+
+apply_at_startup()
+
+from agent.cli.context import AppContext
+from agent.cli.errors import friendly
+from agent.cli import doctor as doctor_cmd
+from agent.cli import eval as eval_cmd
+from agent.cli import eval_claw as eval_claw_cmd
+from agent.cli import eval_compaction as eval_compaction_cmd
+from agent.cli import eval_swe as eval_swe_cmd
+from agent.cli import eval_hle as eval_hle_cmd
+from agent.cli import eval_memory as eval_memory_cmd
+from agent.cli import lessons as lessons_cmd
+from agent.cli import models as model_cmd
+from agent.cli import route as route_cmd
+from agent.cli import sessions as sessions_cmd
+from agent.cli import chat as chat_cmd
+from agent.cli import tui as tui_cmd
+
+app = typer.Typer()
+
+@app.callback()
+def bootstrap(ctx: typer.Context, strict: Annotated[bool, typer.Option()] = False) -> None:
+    ctx.obj = AppContext(strict=strict)
+
+# `friendly` is applied here, once, rather than as a decorator on each command
+# module. One registration site means a new command cannot forget it, and the
+# command modules stay free of CLI exit-code concerns.
+for _name, _fn in (
+    ("doctor", doctor_cmd.doctor),
+    ("models", model_cmd.models),
+    ("route", route_cmd.route),
+    ("chat", chat_cmd.chat),
+    ("tui", tui_cmd.tui),
+    ("eval", eval_cmd.eval_cmd),
+    ("eval-memory", eval_memory_cmd.eval_memory_cmd),
+    ("eval-hle", eval_hle_cmd.eval_hle_cmd),
+    ("eval-claw", eval_claw_cmd.eval_claw_cmd),
+    ("eval-compaction", eval_compaction_cmd.eval_compaction_cmd),
+    ("eval-swe", eval_swe_cmd.eval_swe_cmd),
+    ("lessons", lessons_cmd.lessons_cmd),
+    ("sessions", sessions_cmd.sessions_cmd),
+):
+    app.command(_name)(friendly(_fn))
+
+if __name__ == "__main__":
+    app()