world-model-optimizer 0.2.0__py3-none-any.whl

wmo/cli/e2b_cmds.py

@@ -0,0 +1,212 @@
+"""`wmo e2b reap`: reclaim E2B sandbox slots held by orphaned harbor trial sandboxes.
+
+The operator-facing half of `wmo.harness.e2b_reap`. It renders the reap candidates and their
+evidence, and it never kills anything without `--yes`: a dry run is the default because every
+kill is destructive and one wrong id is somebody's running trial.
+"""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+
+import typer
+from rich.console import Console
+from rich.table import Table
+
+from wmo.harness.e2b_ledger import read_ledger_files
+from wmo.harness.e2b_reap import (
+    AliveSandbox,
+    ReapCandidate,
+    execute_reap,
+    kill_sandbox,
+    list_alive_sandboxes,
+    plan_reap,
+    sandbox_cap,
+)
+
+_console = Console()
+
+
+def _now() -> datetime:
+    """The instant sandbox ages are measured against (a seam tests pin)."""
+    return datetime.now(UTC)
+
+
+e2b_app = typer.Typer(help="Inspect and reclaim E2B sandbox capacity.", no_args_is_help=True)
+
+# Module-level singletons: a typer.Option call cannot be a default inline (ruff B008).
+_REAP_YES = typer.Option(
+    False, "--yes", help="Actually kill the candidates. Without it this is a dry run."
+)
+_REAP_DEAD_OWNERS = typer.Option(
+    True,
+    "--dead-owners/--no-dead-owners",
+    help="Include sandboxes recorded by a wmo run on THIS machine whose process is gone "
+    "(exact ids from the local ledger; safe and on by default).",
+)
+_REAP_STALE_MINUTES = typer.Option(
+    None,
+    "--stale-minutes",
+    min=1,
+    help="Also include any harbor trial sandbox on the ACCOUNT started more than N minutes "
+    "ago. This match is account-wide and can kill another machine's live run, so it is "
+    "opt-in; sandboxes owned by a live local process are still excluded.",
+)
+
+
+@e2b_app.command("reap")
+def reap(
+    yes: bool = _REAP_YES,
+    dead_owners: bool = _REAP_DEAD_OWNERS,
+    stale_minutes: int | None = _REAP_STALE_MINUTES,
+) -> None:
+    """Free E2B concurrency slots held by sandboxes no run is using any more.
+
+    A harbor trial sandbox keeps its slot until its own multi-hour timeout, so a run that dies
+    without graceful shutdown (crash, SIGKILL, budget abort, machine sleep) leaves orphans that
+    starve every later run at the account cap of 100 concurrent sandboxes (override with
+    `$WMO_E2B_SANDBOX_CAP`).
+
+    Two evidence classes, most conservative first:
+
+    - `--dead-owners` (default): unreleased entries in this machine's sandbox ledger whose
+      owning process is gone. These are provably orphans of local runs and are killed by exact
+      recorded id.
+    - `--stale-minutes N` (opt-in): every sandbox ON THE ACCOUNT that carries harbor trial
+      metadata and started more than N minutes ago. The match is account-wide, so it can kill
+      a run on another machine or in another checkout; use it only when you know no such run
+      should be alive. Sandboxes whose local owner process is still running are never selected.
+
+    The default is a dry run that prints the candidates and changes nothing. Pass `--yes` to
+    kill them.
+    """
+    try:
+        cap = sandbox_cap()
+    except ValueError as error:
+        raise typer.BadParameter(str(error)) from error
+    if not dead_owners and stale_minutes is None:
+        raise typer.BadParameter(
+            "--no-dead-owners with no --stale-minutes selects nothing; drop --no-dead-owners "
+            "or add --stale-minutes N"
+        )
+    try:
+        alive = list_alive_sandboxes()
+    except ImportError as error:
+        raise typer.BadParameter(str(error)) from error
+    except Exception as error:  # noqa: BLE001 - any provider failure is a usage-level message
+        raise typer.BadParameter(
+            f"could not list E2B sandboxes ({type(error).__name__}: {error}); check "
+            "$E2B_API_KEY and your connection"
+        ) from error
+
+    plan = plan_reap(
+        alive=alive,
+        ledger_files=read_ledger_files(),
+        now=_now(),
+        dead_owners=dead_owners,
+        stale_minutes=stale_minutes,
+    )
+    _print_usage(alive, cap)
+    if not plan.candidates:
+        _console.print("[green]nothing to reap[/green]: no orphaned sandbox matched")
+        if not plan.vanished:
+            return
+        if not yes:
+            _console.print(
+                f"{len(plan.vanished)} ledger record(s) name sandboxes E2B no longer has; "
+                "--yes clears them"
+            )
+            return
+        outcome = execute_reap(plan, killer=kill_sandbox)
+        _console.print(
+            f"released {len(plan.vanished)} stale ledger record(s); "
+            f"pruned {len(outcome.pruned_ledgers)} ledger file(s)"
+        )
+        return
+
+    _console.print(_candidates_table(plan.candidates))
+    if not yes:
+        _console.print(
+            f"[yellow]dry run[/yellow]: {len(plan.candidates)} sandbox(es) would be killed, "
+            f"freeing up to {len(plan.candidates)} of {cap} slot(s). Re-run with --yes to do it."
+        )
+        return
+
+    outcome = execute_reap(plan, killer=kill_sandbox)
+    remaining = max(len(alive) - outcome.freed, 0)
+    _console.print(
+        f"[green]reaped[/green] {outcome.freed} sandbox(es); usage now {remaining}/{cap} "
+        f"({max(cap - remaining, 0)} free)"
+    )
+    if outcome.already_gone:
+        _console.print(
+            f"{len(outcome.already_gone)} candidate(s) were already gone: "
+            f"{', '.join(outcome.already_gone)}"
+        )
+    for sandbox_id, error in outcome.failed:
+        _console.print(f"[red]kill failed[/red] {sandbox_id}: {error}")
+    if outcome.pruned_ledgers:
+        _console.print(f"pruned {len(outcome.pruned_ledgers)} fully released ledger file(s)")
+
+
+def _print_usage(alive: list[AliveSandbox], cap: int) -> None:
+    """Print current account usage against the cap."""
+    trials = sum(1 for sandbox in alive if sandbox.is_harbor_trial())
+    _console.print(
+        f"E2B usage: [bold]{len(alive)}/{cap}[/bold] concurrent sandbox(es) running "
+        f"({trials} harbor trial environment(s)), {max(cap - len(alive), 0)} slot(s) free"
+    )
+
+
+def _candidates_table(candidates: tuple[ReapCandidate, ...]) -> Table:
+    """Render one row per reap candidate with the evidence behind it."""
+    table = Table(title="Reap candidates")
+    table.add_column("Sandbox", no_wrap=True)
+    table.add_column("Age", justify="right", no_wrap=True)
+    table.add_column("Template", no_wrap=True)
+    table.add_column("Source", no_wrap=True)
+    table.add_column("Owner", justify="right", no_wrap=True)
+    table.add_column("Alive", no_wrap=True)
+    table.add_column("Trial")
+    for candidate in candidates:
+        table.add_row(
+            candidate.sandbox_id,
+            _age(candidate.age_seconds),
+            _template(candidate.template_id),
+            candidate.source,
+            "unknown" if candidate.owner_pid is None else str(candidate.owner_pid),
+            _owner_liveness(candidate.owner_alive),
+            candidate.trial_name or "",
+        )
+    return table
+
+
+# A wmo harbor alias is `wmo-hb-v1-<64 hex>`: printing it whole squeezes every evidence column
+# out of the table, and its head plus tail already identify a template uniquely in practice.
+_TEMPLATE_DISPLAY_LEN = 22
+
+
+def _template(template_id: str) -> str:
+    """Shorten a long template alias, keeping both ends so two aliases stay distinguishable."""
+    if len(template_id) <= _TEMPLATE_DISPLAY_LEN:
+        return template_id
+    return f"{template_id[: _TEMPLATE_DISPLAY_LEN - 8]}…{template_id[-7:]}"
+
+
+def _owner_liveness(owner_alive: bool | None) -> str:
+    """Render owner liveness, distinguishing "no recorded owner" from "owner is gone"."""
+    if owner_alive is None:
+        return "unknown"
+    return "yes" if owner_alive else "no"
+
+
+def _age(seconds: float) -> str:
+    """Human-readable sandbox age, e.g. `3h12m`."""
+    minutes = int(seconds // 60)
+    hours, minutes = divmod(minutes, 60)
+    return f"{hours}h{minutes:02d}m" if hours else f"{minutes}m"
+
+
+def register(app: typer.Typer) -> None:
+    """Attach the `wmo e2b` command group to the root CLI."""
+    app.add_typer(e2b_app, name="e2b")

wmo/cli/eval_closed_loop.py

@@ -0,0 +1,207 @@
+"""`wmo eval --mode closed-loop` and `wmo eval agreement` — the closed-loop halves of eval.
+
+Kept out of `app.py` so the (large) eval command stays readable; `app.py` routes here.
+Closed-loop mode runs an agent harness against a built world model — the environment is ALWAYS
+the world-model simulation; `--harness-backend e2b` only moves the pi-node harness PROCESS into
+pooled E2B sandboxes (its tool calls stay answered host-side) — and scores task success;
+`agreement` compares two saved closed-loop reports — the outcome-agreement check
+docs/reference/closed_loop.md names.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import typer
+from rich.console import Console
+
+from wmo.cli.model_roles import resolve_opt_in_model_provider
+from wmo.config import WorldModelStore
+from wmo.engine import load_world_model
+from wmo.evals.agreement import compute_agreement
+from wmo.evals.closed_loop import ClosedLoopEval, ClosedLoopReport
+from wmo.evals.gold import GoldJudge, GoldVerdict
+from wmo.evals.tasks import load_tasks
+from wmo.harness.doc import MAX_TURNS_ID, HarnessDoc, Surface, SurfaceKind
+from wmo.harness.runtime import DEFAULT_MAX_TURNS, AgentRuntime
+from wmo.harness.store import HarnessStore
+
+
+def run_closed_loop(
+    console: Console,
+    *,
+    tasks_file: str,
+    name: str | None,
+    root: str,
+    k: int,
+    max_turns: int | None,
+    out: str | None,
+    harness: str | None = None,
+    harness_backend: str = "local",
+    eval_concurrency: int | None = None,
+    e2b_template: str | None = None,
+) -> None:
+    """Run an agent harness on each task against the world model; print and optionally save.
+
+    `--harness <name>[@ref]` runs a stored harness version (ref = version or alias; default is
+    the champion alias); without it the built-in baseline loop runs. `max_turns=None` means "the
+    harness's own cap" (or the default for the baseline); an explicit value overrides either —
+    never silently ignored. `[models.agent]` selects the agent model for either path; when unset,
+    the agent shares the world model's provider. The environment is ALWAYS the world-model
+    simulation;
+    `--harness-backend e2b` only moves the pi-node harness PROCESS into pooled E2B sandboxes
+    (tool calls stay answered by the world model host-side), running all (task, attempt) cells
+    at once unless `--eval-concurrency` caps them.
+    """
+    if harness_backend not in ("local", "e2b"):
+        raise typer.BadParameter(
+            f"unknown --harness-backend {harness_backend!r}; choose local or e2b"
+        )
+    try:
+        tasks = load_tasks(tasks_file)
+    except (OSError, ValueError) as exc:  # missing file, malformed JSONL, empty, duplicate ids
+        raise typer.BadParameter(f"cannot load tasks from {tasks_file!r}: {exc}") from exc
+    # The world model IS the environment on every backend, so it is always required.
+    store = WorldModelStore(root)
+    try:
+        model_dir = store.resolve(name)
+    except (FileNotFoundError, ValueError) as exc:
+        raise typer.BadParameter(str(exc)) from exc
+    world_model, provider = load_world_model(model_dir)
+    agent_provider, agent_model = resolve_opt_in_model_provider(root, "agent", provider)
+
+    loaded_harness = _load_harness(harness, root)
+    if loaded_harness is None and harness_backend == "e2b":
+        raise typer.BadParameter(
+            "--harness-backend e2b runs a pi-node harness process in sandboxes; the built-in "
+            "baseline loop has no such process — pass --harness"
+        )
+    agent_label = (
+        f"{loaded_harness.name}-v{loaded_harness.version}"
+        if loaded_harness is not None
+        else "baseline"
+    )
+    agent_identity = (
+        f"{agent_provider.config.kind.value}:{agent_model}" if agent_model is not None else None
+    )
+    report_agent_label = (
+        f"{agent_label}[agent={agent_identity}]" if agent_identity is not None else agent_label
+    )
+    agent_note = (
+        f" using agent model [bold]{agent_identity}[/bold]" if agent_identity is not None else ""
+    )
+    versus = (
+        f"world model [bold]{model_dir.name}[/bold]"
+        if harness_backend == "local"
+        else f"world model [bold]{model_dir.name}[/bold] (pi harness in pooled E2B sandboxes)"
+    )
+    console.print(
+        f"closed-loop: harness [bold]{agent_label}[/bold]{agent_note} vs {versus} "
+        f"on {len(tasks)} task(s), k={k}…"
+    )
+
+    def _progress(task_id: str, attempt: int, verdict: GoldVerdict) -> None:
+        mark = "[green]pass[/green]" if verdict.passed else "[red]fail[/red]"
+        console.print(f"  {task_id} #{attempt}: {mark} ({verdict.rationale})")
+
+    if loaded_harness is not None:
+        if (
+            harness_backend == "local"
+            and loaded_harness.runtime_kind() == "pi-node"
+            and eval_concurrency is not None
+            and eval_concurrency != 1
+        ):
+            # Local pi runtimes are single-episode resources (one runner port/workdir, or one
+            # RunnerLink channel): parallel cells would collide.
+            raise typer.BadParameter(
+                "pi-node harnesses run one episode at a time under --harness-backend local; "
+                "drop --eval-concurrency or use --harness-backend e2b"
+            )
+        if max_turns is not None and max_turns != loaded_harness.max_turns():
+            console.print(
+                f"  note: --max-turns {max_turns} overrides the harness's own "
+                f"max_turns={loaded_harness.max_turns()}"
+            )
+            loaded_harness = _with_max_turns(loaded_harness, max_turns)
+        try:
+            runtime = loaded_harness.runtime(
+                agent_provider,
+                backend=harness_backend,
+                e2b_template=e2b_template,
+            )
+        except ValueError as exc:  # e.g. e2b on a non-pi-node harness -> usage error
+            raise typer.BadParameter(str(exc)) from exc
+    else:
+        runtime = AgentRuntime(agent_provider, max_turns=max_turns or DEFAULT_MAX_TURNS)
+    try:
+        evaluation = ClosedLoopEval(
+            tasks,
+            world_model,
+            agent_provider,
+            GoldJudge(provider),
+            label=f"{report_agent_label}@{model_dir.name}",
+            k=k,
+            concurrency=(
+                eval_concurrency
+                if eval_concurrency is not None
+                else (0 if harness_backend == "e2b" else 1)
+            ),
+            runtime=runtime,
+            on_progress=_progress,
+        )
+        report = evaluation.run()
+    finally:
+        if harness_backend == "e2b":
+            # An eval-owned e2b runtime owns a private sandbox pool; tear it down with the eval.
+            from wmo.harness.pi_e2b import E2BPiRuntime
+
+            if isinstance(runtime, E2BPiRuntime):
+                runtime.close()
+    for task_id, outcome in report.per_task.items():
+        console.print(
+            f"  {task_id:24} success={outcome.success_rate:.2f} "
+            f"assertions={outcome.mean_fraction:.2f}"
+        )
+    console.print(f"[bold]OVERALL[/bold] {report.summary()}")
+    if out:
+        Path(out).write_text(report.model_dump_json(indent=2), encoding="utf-8")
+        console.print(f"wrote closed-loop report -> {out}")
+
+
+def run_agreement(console: Console, *, report_a: str, report_b: str, threshold: float) -> None:
+    """Compare two saved closed-loop reports task-by-task and print the agreement verdict."""
+    a = _load_report(report_a)
+    b = _load_report(report_b)
+    result = compute_agreement(a, b, pass_threshold=threshold)
+    c = result.confusion
+    la, lb = result.label_a or "A", result.label_b or "B"
+    console.print(f"[bold]task verdict confusion[/bold] ({la} vs {lb}):")
+    console.print(f"  {la}-pass & {lb}-pass: {c.a_pass_b_pass}")
+    console.print(f"  {la}-pass & {lb}-FAIL: {c.a_pass_b_fail}  (A over-credits these)")
+    console.print(f"  {la}-FAIL & {lb}-pass: {c.a_fail_b_pass}")
+    console.print(f"  {la}-FAIL & {lb}-FAIL: {c.a_fail_b_fail}")
+    console.print(f"[bold]VERDICT[/bold] {result.summary()}")
+
+
+def _load_report(path: str) -> ClosedLoopReport:
+    try:
+        return ClosedLoopReport.model_validate_json(Path(path).read_text(encoding="utf-8"))
+    except (OSError, ValueError) as exc:
+        raise typer.BadParameter(f"cannot read closed-loop report {path!r}: {exc}") from exc
+
+
+def _with_max_turns(doc: HarnessDoc, max_turns: int) -> HarnessDoc:
+    """A copy of `doc` with its max-turns surface replaced (re-validated via the constructor)."""
+    surfaces = [s for s in doc.surfaces if s.id != MAX_TURNS_ID]
+    surfaces.append(Surface(id=MAX_TURNS_ID, kind=SurfaceKind.PARAM, content=str(max_turns)))
+    return HarnessDoc(name=doc.name, version=doc.version, surfaces=surfaces)
+
+
+def _load_harness(name: str | None, root: str) -> HarnessDoc | None:
+    if name is None:
+        return None
+    base, _, ref = name.partition("@")
+    try:
+        return HarnessStore(root).load(base, ref or None)
+    except (FileNotFoundError, ValueError) as exc:
+        raise typer.BadParameter(str(exc)) from exc