tare-cli 0.2.0__py3-none-any.whl

tare/calibrate.py

@@ -0,0 +1,98 @@
+"""Calibrate: an agent's pass rate on a task, before Cliff or Swap spend tails on it (epic Calibrate).
+
+Each side (an agent with its arguments, e.g. `claude --model haiku`) gets N fresh starts of
+the task in rooms, scored by the check. The report gives each rate with its interval, so a
+task can be chosen where one side mostly fails and the other mostly passes.
+
+A side may have its own prompt, so two skills invoked differently compare in one call. Without
+a check every run that ends counts as finished, and the workspaces are kept for scoring
+elsewhere.
+"""
+import re
+import shutil
+import statistics
+import time
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from . import capsule as caps
+from . import usage as usages
+from .cliff import wilson
+from .journal import Journal
+
+
+@dataclass
+class Side:
+    key: str
+    agent: object
+    real: object
+    args: list[str]
+    prompt: str | None = None  # the side's own prompt; None: the shared one
+    results: list[bool] = field(default_factory=list)
+    scores: list[int] = field(default_factory=list)  # judge scores, where the check is a judge
+    times: list[float] = field(default_factory=list)  # seconds per run
+
+    def label(self) -> str:
+        return " ".join([self.agent.name, *self.args])
+
+
+def _minutes(seconds: float) -> str:
+    return f"{int(seconds // 60)}:{int(seconds % 60):02d}"
+
+
+def calibrate(sides: list[Side], project: Path, prompt: str | None, check: str | None, out: Path, *, runs: int = 10,
+              jobs: int = 3, env: dict[str, str] | None = None, journal: Journal | None = None,
+              keep: bool = False) -> str:
+    journal = journal or Journal(None)
+    keep = keep or check is None  # without a check the workspaces are what is scored
+    journal("start", kind="calibrate", task=prompt, check=check, project=str(project), tails=runs,
+            sides={s.key: {"agent": s.agent.name, "args": s.args, "dir": ".", "prompt": s.prompt,
+                           "billing": usages.billing(s.agent, s.real)} for s in sides})
+    work = out / "project"
+    shutil.copytree(project, work, symlinks=True)
+    caps.archive(work, out / "start.tar")
+    shutil.rmtree(work)
+    start = caps.Capsule(0, None, out / "start.tar", 0, 0, "start")
+    journal("phase", phase="runs")
+
+    def run(job: tuple[Side, int]) -> tuple[Side, caps.Tail]:
+        side, i = job
+        name = f"{side.key}-{i}"
+        journal("tail", id=name, status="running", side=side.key, agent=side.key, agent_name=side.agent.name,
+                kind="fresh", dir=f"calibrate/{name}")
+        began = time.monotonic()
+        tail = caps.run_tail(side.agent, side.real, out, start, side.prompt or prompt, side.args, check, i, env,
+                             tail_dir=out / "calibrate" / name, keep=keep)
+        journal("tail", id=name, status="passed" if tail.passed else "failed", detail=tail.detail,
+                usage=tail.usage, check_usage=tail.check_usage)
+        return side, tail, time.monotonic() - began
+
+    with ThreadPoolExecutor(max_workers=jobs) as pool:
+        for side, tail, seconds in pool.map(run, [(s, i) for i in range(runs) for s in sides]):
+            side.results.append(tail.passed)
+            side.times.append(seconds)
+            if score := re.search(r"score (\d+)", tail.detail):
+                side.scores.append(int(score.group(1)))
+
+    width = max(len(s.label()) for s in sides) + 2
+    lines = [f"tare calibrate · {project}"]
+    if prompt and not all(s.prompt for s in sides):
+        lines.append(f"  task      {prompt}")
+    lines += [f"  task {s.key:<4} {s.prompt}" for s in sides if s.prompt]
+    lines += [f"  check     {check or 'none. A run counts as finished when its agent exits 0. Its workspace is kept for scoring elsewhere.'}",
+              "", f"  side  {'agent':<{width}}" + ("pass  rate  95% interval" if check else "finished  time (median, range)")]
+    for s in sides:
+        x, n = sum(s.results), len(s.results)
+        if check is None:
+            lines.append(f"  {s.key:<4}  {s.label():<{width}}{x:>4}/{n:<4}  {_minutes(statistics.median(s.times))}"
+                         f" ({_minutes(min(s.times))}-{_minutes(max(s.times))})")
+            continue
+        lo, hi = wilson(x, n)
+        lines.append(f"  {s.key:<4}  {s.label():<{width}}{x:>2}/{n:<2} {x / n:.2f}  {lo:.2f}-{hi:.2f}"
+                     + (f"   scores {' '.join(map(str, sorted(s.scores)))}" if s.scores else ""))
+    text = "\n".join(lines + usages.report(out)) + "\n"
+    (out / "report.txt").write_text(text)
+    journal("report", text=text)
+    journal("phase", phase="done")
+    return text

tare/capsule.py

@@ -0,0 +1,246 @@
+"""Capsules: a run in the room, resumable at every step (epics Cliff and Swap).
+
+During the original run a PostToolUse hook archives /work after every tool call into a
+run directory outside the room. A step is one model tool call, or several issued together;
+its capsule is the archive taken after its last tool plus the session up to its results.
+Capsule 0 is the project before the run. A capsule is continued either natively (the
+agent's own session, cut after the step; Claude Code) or by handoff (a fresh session
+started with the trail rendered up to the step; every agent).
+"""
+import hashlib
+import json
+import os
+import shutil
+import subprocess
+import tarfile
+from dataclasses import dataclass
+from pathlib import Path
+
+from . import trail as trails
+from . import usage as usages
+from .room import ROOM_HOME, bwrap, room_home
+
+RUN_MOUNT = "/tare-run"
+HOOK_COMMAND = f"/bin/sh {RUN_MOUNT}/hook.sh"
+# Archives /work after every tool call, numbered, named by the call's tool_use_id.
+# The hook input is JSON on stdin; its top-level tool_use_id is the first one in it.
+HOOK = """#!/bin/sh
+id=$(grep -o '"tool_use_id": *"[^"]*"' | head -1 | sed 's/.*"\\([^"]*\\)"$/\\1/')
+n=$(ls /tare-run/capsules | wc -l)
+tar -C /work -cf "/tare-run/capsules/$(printf %04d "$n")-$id.tar" .
+"""
+CONTINUE = "Continue."
+
+
+@dataclass
+class Capsule:
+    step: int
+    tool_use_id: str | None  # None for step 0
+    archive: Path  # tar of /work; identical workspaces share one archive
+    cut: int  # native session lines kept (0 for step 0, or when there is no native resume)
+    trail_cut: int  # trail events up to this step
+    tool: str  # what the step did, for reports
+
+
+@dataclass
+class Tail:
+    step: int
+    passed: bool
+    detail: str
+    usage: dict | None = None  # the agent's tokens and cost
+    check_usage: dict | None = None  # what the check spent, for example a judge agent
+
+
+def _contents(path: Path) -> dict[str, str]:
+    """Path -> content digest of every file in a capsule archive (not its timestamps)."""
+    with tarfile.open(path) as tar:
+        return {m.name.removeprefix("./"): hashlib.sha256(tar.extractfile(m).read()).hexdigest()
+                for m in tar.getmembers() if m.isfile()}
+
+
+def _digest(path: Path) -> str:
+    return hashlib.sha256(json.dumps(sorted(_contents(path).items())).encode()).hexdigest()
+
+
+def archive(directory: Path, target: Path):
+    with tarfile.open(target, "w") as tar:
+        tar.add(directory, arcname=".")
+
+
+def unpack(source: Path, directory: Path):
+    with tarfile.open(source) as tar:
+        tar.extractall(directory, filter="tar")
+
+
+def _config(agent, home: Path) -> Path:
+    return home / Path(agent.room_config).relative_to(ROOM_HOME)
+
+
+def record(agent, real, project: Path, out: Path, prompt: str, args: list[str],
+           env: dict[str, str] | None = None) -> int:
+    """The original run, in a room, on a copy of the project, with a capsule per tool call.
+    Returns the agent's exit code."""
+    store = out / "store"
+    (store / "capsules").mkdir(parents=True)
+    (store / "hook.sh").write_text(HOOK)
+    work = out / "original" / "work"
+    shutil.copytree(project, work, symlinks=True)
+    archive(work, store / "capsules" / "0000-start.tar")
+    (out / "original" / "task.txt").write_text(prompt)
+    with room_home(agent, real) as home:
+        agent.prepare_hook(_config(agent, home), HOOK_COMMAND)
+        command = [agent.name, *agent.room_flags, *agent.run_args(prompt, args, hook=HOOK_COMMAND)]
+        code = _stream(bwrap(agent, real, home, work, command, env, ["--bind", str(store), RUN_MOUNT]),
+                       out / "original", None)
+        session = agent.session_file(home)
+        if session:
+            shutil.copyfile(session, out / "original" / "session.jsonl")
+    return code
+
+
+def _stream(argv: list[str], log_dir: Path, timeout: float | None) -> int:
+    """Run a room with the agent's stdout and stderr written to log_dir as they come (the dashboard reads them)."""
+    with (log_dir / "stdout.jsonl").open("w") as out, (log_dir / "stderr.log").open("w") as err:
+        proc = subprocess.Popen(argv, stdout=out, stderr=err, stdin=subprocess.DEVNULL)
+        try:
+            return proc.wait(timeout=timeout)
+        except subprocess.TimeoutExpired:
+            proc.kill()
+            proc.wait()
+            raise
+
+
+def session_lines(out: Path) -> list[str]:
+    path = out / "original" / "session.jsonl"
+    return path.read_text().splitlines() if path.exists() else []
+
+
+def capsules(out: Path, agent) -> list[Capsule]:
+    """The resumable steps of the original run, in order."""
+    archives = sorted((out / "store" / "capsules").glob("*.tar"))
+    by_id = {path.stem.split("-", 1)[1]: path for path in archives[1:]}
+    result = [Capsule(0, None, archives[0], 0, 0, "start")]
+    seen = {_digest(archives[0]): archives[0]}
+    previous = archives[0]
+    for step in agent.trail(session_lines(out)).steps:
+        taken = [by_id[i] for i in step.ids if i in by_id]
+        # the last snapshot of the step; a step whose tools the hook never saw left the workspace as it was
+        path = max(taken) if taken else previous
+        shared = seen.setdefault(_digest(path), path)
+        if shared != path:
+            path.unlink()  # same workspace as an earlier capsule: keep one archive
+        result.append(Capsule(len(result), step.ids[-1], shared, step.native_cut or 0, step.trail_cut, step.tool))
+        previous = shared
+    return result
+
+
+def session_id(out: Path) -> str:
+    for line in session_lines(out):
+        entry = json.loads(line)
+        # Claude Code: sessionId on every entry; Codex: the session_meta payload; Pi: the session header
+        sid = (entry.get("sessionId") or (entry.get("payload") or {}).get("id")
+               or (entry.get("id") if entry.get("type") == "session" else None))
+        if sid:
+            return sid
+    raise ValueError("no session id in the session file")
+
+
+def _run_in(agent, real, home: Path, work: Path, command: list[str], env, timeout: float, log_dir: Path) -> str:
+    try:
+        return f"agent exit {_stream(bwrap(agent, real, home, work, command, env), log_dir, timeout)}"
+    except subprocess.TimeoutExpired:
+        return f"agent stopped after {int(timeout)} s"
+
+
+def read_usage(agent, log_dir: Path) -> dict | None:
+    """The tokens and cost of the run whose stream is in log_dir (None for an agent that cannot tell)."""
+    stream = log_dir / "stdout.jsonl"
+    if not hasattr(agent, "usage"):
+        return None
+    return agent.usage(stream.read_text(errors="replace") if stream.exists() else "").to_dict()
+
+
+def read_check_usage(path: Path) -> dict | None:
+    if not path.exists():
+        return None
+    found = usages.total(json.loads(line) for line in path.read_text().splitlines() if line.strip())
+    return found.to_dict() if found else None
+
+
+def _finish(tail_dir: Path, work: Path, step: int, ran: str, check: str, extra: dict | None = None,
+            keep: bool = False, usage: dict | None = None) -> Tail:
+    # without a check (calibrate only) a run counts when its agent ended normally
+    log = tail_dir / "check-usage.jsonl"
+    passed, detail = run_check(check, work, usage_log=log) if check else (ran == "agent exit 0", "no check")
+    check_usage = read_check_usage(log)
+    (tail_dir / "result.json").write_text(json.dumps({"step": step, "passed": passed, "agent": ran, "check": detail,
+                                                      "usage": usage, "check_usage": check_usage, **(extra or {})}))
+    if not keep:
+        shutil.rmtree(work, ignore_errors=True)
+    return Tail(step, passed, f"{ran}; {detail}", usage, check_usage)
+
+
+def run_tail(agent, real, out: Path, capsule: Capsule, prompt: str, args: list[str], check: str,
+             index: int, env: dict[str, str] | None = None, timeout: float = 1800, tail_dir: Path | None = None,
+             keep: bool = False) -> Tail:
+    """Continue a capsule natively in a fresh room (the agent's own session, cut after the step),
+    run it to the end, score the workspace with the check. Step 0 is a fresh start."""
+    tail_dir = tail_dir or out / "tails" / f"{capsule.step:04d}-{index}"
+    work = tail_dir / "work"
+    work.mkdir(parents=True)
+    unpack(capsule.archive, work)
+    with room_home(agent, real) as home:
+        if capsule.step == 0:
+            command = [agent.name, *agent.room_flags, *agent.run_args(prompt, args)]
+        else:
+            sid = session_id(out)
+            agent.place_session(home, sid, session_lines(out)[:capsule.cut])
+            command = [agent.name, *agent.room_flags, *agent.resume_args(sid, CONTINUE, args)]
+        ran = _run_in(agent, real, home, work, command, env, timeout, tail_dir)
+    return _finish(tail_dir, work, capsule.step, ran, check, keep=keep, usage=read_usage(agent, tail_dir))
+
+
+def check_original(agent, out: Path, check: str) -> tuple[bool, str, dict]:
+    """The check on the original run's workspace, and the usage of that run and of its check."""
+    log = out / "original" / "check-usage.jsonl"
+    passed, detail = run_check(check, out / "original" / "work", usage_log=log)
+    return passed, detail, {"usage": read_usage(agent, out / "original"), "check_usage": read_check_usage(log)}
+
+
+def run_handoff(agent, real, tail_dir: Path, capsule: Capsule, events: list[trails.Event], task: str,
+                args: list[str], check: str, env: dict[str, str] | None = None, workspace_only: bool = False,
+                timeout: float = 1800) -> Tail:
+    """Continue a capsule by handoff: a fresh session of any agent, in the capsule's workspace,
+    started with the trail rendered up to the step."""
+    work = tail_dir / "work"
+    work.mkdir(parents=True)
+    unpack(capsule.archive, work)
+    handoff = trails.render(task, events[:capsule.trail_cut], workspace_only)
+    (tail_dir / "handoff.txt").write_text(handoff)
+    with room_home(agent, real) as home:
+        ran = _run_in(agent, real, home, work, [agent.name, *agent.room_flags, *agent.run_args(handoff, args)],
+                      env, timeout, tail_dir)
+    return _finish(tail_dir, work, capsule.step, ran, check, {"agent_name": agent.name},
+                   usage=read_usage(agent, tail_dir))
+
+
+# a check that runs an agent itself (tare judge) appends that agent's usage to this file
+USAGE_LOG = "TARE_USAGE_LOG"
+
+
+def run_check(check: str, work: Path, timeout: float = 600, usage_log: Path | None = None) -> tuple[bool, str]:
+    """The user's check, outside the room: exit code 0 passes."""
+    env = {**os.environ, USAGE_LOG: str(usage_log)} if usage_log else None
+    try:
+        proc = subprocess.run(check, shell=True, cwd=work, capture_output=True, text=True, timeout=timeout,
+                              stdin=subprocess.DEVNULL, env=env)
+    except subprocess.TimeoutExpired:
+        return False, f"check timed out after {int(timeout)} s"
+    last = next((line.strip() for line in reversed(proc.stdout.splitlines()) if line.strip()), "")
+    return proc.returncode == 0, f"check exit {proc.returncode}" + (f": {last[:120]}" if last else "")
+
+
+def changed_files(before: Path, after: Path) -> list[str]:
+    """Paths whose content differs between two capsule archives."""
+    a, b = _contents(before), _contents(after)
+    return sorted(name for name in a.keys() | b.keys() if a.get(name) != b.get(name))

tare/cli.py

@@ -0,0 +1,312 @@
+"""tare: zero the scale before you weigh.
+
+  tare probe {claude,codex,pi,agy} [--project DIR]
+  tare {claude,codex,pi,agy} [--project DIR] [--allow-dirty] [--yolo] [-- AGENT_ARGS...]
+  tare cliff {claude,codex,pi,agy} PROMPT --check CMD [--tails N] [--budget N] [--jobs N] [-- AGENT_ARGS...]
+  tare swap PROMPT --check CMD [--a claude] [--b codex] [--a-args ARGS] [--b-args ARGS] [--cuts 0,0.5,1]
+  tare calibrate [PROMPT] [--check CMD] --side "claude --model haiku" --side "codex" [--side-prompt b=TEXT] [--runs N]
+  tare judge [DIR] --rubric FILE --threshold N [--judge "claude --model sonnet"]   a check: exit 0 at or above N
+  tare judge-noise DIR... --rubric FILE --times K [--threshold N]                  how far the judge's scores vary
+  tare watch DIR... [--port N]   the dashboard of a run, live or finished (several runs on one page)
+  tare rerun DIR [--out DIR]     repeat a run from its recipe
+  tare prices [update]           the price tables for costs that an agent does not report
+"""
+import argparse
+import json
+import shlex
+import threading
+import signal
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+from . import dashboard, recipe
+from . import usage as usages
+from .agents import AGENTS
+from . import calibrate as calibration
+from .cliff import cliff
+from .journal import Journal
+from .swap import Side, swap
+from .probe import probe, render
+from .room import TareError, bwrap, room_home
+
+
+def _exit_on(signum, _frame):
+    # SystemExit unwinds through room_home, so the login copy is removed
+    raise SystemExit(128 + signum)
+
+
+def _run_attached(argv: list[str]) -> int:
+    """Run the room in the foreground. Ctrl-C belongs to the agent (it interrupts a
+    turn there), so tare ignores it instead of dying and removing the room under it."""
+    previous = signal.signal(signal.SIGINT, signal.SIG_IGN)
+    try:
+        return subprocess.call(argv)
+    finally:
+        signal.signal(signal.SIGINT, previous)
+
+
+def _serve(out: Path | list[Path], port: int) -> str:
+    try:
+        return dashboard.serve(out, port)[1]
+    except OSError:
+        return dashboard.serve(out, 0)[1]  # the usual port is taken
+
+
+def main(argv: list[str] | None = None) -> int:
+    argv = list(sys.argv[1:] if argv is None else argv)
+    original_argv = list(argv)
+    if argv[:1] == ["watch"]:
+        return _watch(argv[1:])
+    if argv[:1] == ["rerun"]:
+        return _rerun(argv[1:])
+    if argv[:1] == ["prices"]:
+        return _prices(argv[1:])
+    if argv[:1] in (["judge"], ["judge-noise"]):
+        return _judge(argv[0], argv[1:])
+    passthrough: list[str] = []
+    if "--" in argv:
+        at = argv.index("--")
+        argv, passthrough = argv[:at], argv[at + 1:]
+
+    ap = argparse.ArgumentParser(prog="tare", description="Start a coding agent in an isolated room without your "
+                                 "personal setup, and prove the room clean before the run.")
+    ap.add_argument("--version", action="version", version=f"tare {recipe.tare_version()}")
+    sub = ap.add_subparsers(dest="command", required=True)
+    p = sub.add_parser("probe", help="probe the room: tare: 0.00, the leaks and their sources, or not proven (blind)")
+    p.add_argument("agent", choices=sorted(AGENTS))
+    p.add_argument("--project", type=Path, default=Path.cwd())
+    for name in sorted(AGENTS):
+        c = sub.add_parser(name, help=f"probe, then start {name} in the room (args after --)")
+        c.add_argument("--project", type=Path, default=Path.cwd())
+        c.add_argument("--allow-dirty", action="store_true", help="start even if the reading is not tare: 0.00")
+        c.add_argument("--yolo", action="store_true", help="skip the agent's permission prompts (for Codex also its own sandbox). The room stays.")
+    k = sub.add_parser("cliff", help="find where a failed run became lost (args after -- go to every agent run)")
+    k.add_argument("agent", choices=sorted(AGENTS))
+    k.add_argument("prompt", help="the task, as given to the agent")
+    k.add_argument("--check", required=True, help="shell command run in the finished workspace (exit 0 passes)")
+    k.add_argument("--project", type=Path, default=Path.cwd())
+    k.add_argument("--tails", type=int, default=3, help="tails (continuations from a saved step) per step tried (default 3)")
+    k.add_argument("--budget", type=int, default=30, help="tails in total, baseline included (default 30)")
+    k.add_argument("--jobs", type=int, default=3, help="tails run at the same time (default 3)")
+    k.add_argument("--gap-below", type=float, default=0.2,
+                   help="report a model gap (the task is too hard for the model) only when the upper end of the "
+                        "baseline's 95%% interval is below this (default 0.2)")
+    k.add_argument("--out", type=Path, help="run directory (default ~/.local/state/tare/cliff/<project>-<time>)")
+    k.add_argument("--allow-dirty", action="store_true", help="run even if the reading is not tare: 0.00")
+    w = sub.add_parser("swap", help="each agent continues both workspaces: was it the workspace or the agent")
+    w.add_argument("prompt", help="the task, as given to both agents")
+    w.add_argument("--check", required=True, help="shell command run in the finished workspace (exit 0 passes)")
+    w.add_argument("--a", default="claude", choices=sorted(AGENTS), help="agent a (default claude)")
+    w.add_argument("--b", default="codex", choices=sorted(AGENTS), help="agent b (default codex)")
+    w.add_argument("--a-args", default="", help="arguments for every run of agent a, e.g. '--model sonnet'")
+    w.add_argument("--b-args", default="", help="arguments for every run of agent b, e.g. '-m MODEL'")
+    w.add_argument("--cuts", default="0,0.5,1", help="where to cut each run, as fractions of its steps (default 0,0.5,1)")
+    w.add_argument("--tails", type=int, default=3, help="tails (continuations) per cut, workspace and agent (default 3)")
+    w.add_argument("--jobs", type=int, default=3, help="tails run at the same time (default 3)")
+    w.add_argument("--handoff", choices=["trail", "workspace"], default="trail",
+                   help="what a continuing agent gets besides the workspace: a written summary of the steps so far "
+                        "(trail, the default) or nothing (workspace)")
+    w.add_argument("--project", type=Path, default=Path.cwd())
+    w.add_argument("--out", type=Path, help="run directory (default ~/.local/state/tare/swap/<project>-<time>)")
+    w.add_argument("--allow-dirty", action="store_true", help="run even if a reading is not tare: 0.00")
+    c = sub.add_parser("calibrate", help="how often each side (an agent with its arguments) passes a task")
+    c.add_argument("prompt", nargs="?", help="the task, as given to every side without a prompt of its own")
+    c.add_argument("--check", help="shell command run in the finished workspace (exit 0 passes). Without it, a run "
+                   "counts as finished when its agent exits 0, and its workspace is kept for scoring elsewhere")
+    c.add_argument("--side", action="append", required=True,
+                   help="an agent and its arguments, e.g. 'claude --model haiku' (repeat for each side)")
+    c.add_argument("--side-prompt", action="append", default=[], metavar="KEY=PROMPT",
+                   help="a side's own prompt. Sides are a, b, c... in --side order, e.g. 'b=/b:ideate a todo app'")
+    c.add_argument("--runs", type=int, default=10, help="fresh starts per side (default 10)")
+    c.add_argument("--jobs", type=int, default=3, help="runs at the same time (default 3)")
+    c.add_argument("--project", type=Path, default=Path.cwd())
+    c.add_argument("--out", type=Path, help="run directory (default ~/.local/state/tare/calibrate/<project>-<time>)")
+    c.add_argument("--allow-dirty", action="store_true", help="run even if a reading is not tare: 0.00")
+    c.add_argument("--keep", action="store_true", help="keep every run's finished workspace (e.g. for judge-noise)")
+    for parser in (k, w, c):
+        parser.add_argument("--port", type=int, default=8777, help="dashboard port on localhost (default 8777)")
+    sub.add_parser("watch", help="the live dashboard of one or more run directories (tare watch DIR...)")
+    sub.add_parser("judge", help="score a page by a rubric, in a room without agent or model names so the judge cannot tell who "
+                       "built it (usable as --check)")
+    sub.add_parser("judge-noise", help="score the same pages repeatedly, to see how far the judge's scores vary")
+    sub.add_parser("rerun", help="repeat a run from its recipe (tare rerun DIR)")
+    sub.add_parser("prices", help="the price tables for costs that an agent does not report (tare prices update)")
+    args = ap.parse_args(argv)
+    if args.command == "swap":
+        names = [args.a, args.b]
+    elif args.command == "calibrate":
+        sides = [shlex.split(side) for side in args.side]
+        unknown = [side[0] for side in sides if side[0] not in AGENTS]
+        if unknown:
+            ap.error(f"unknown agent: {', '.join(unknown)} (known: {', '.join(sorted(AGENTS))})")
+        names = [side[0] for side in sides]
+        keys = [chr(97 + i) for i in range(len(sides))]
+        prompts = dict(item.split("=", 1) for item in args.side_prompt if "=" in item)
+        bad = [item for item in args.side_prompt if "=" not in item or item.split("=", 1)[0] not in keys]
+        if bad:
+            ap.error(f"--side-prompt wants KEY=PROMPT with KEY one of {', '.join(keys)}: {', '.join(bad)}")
+        if not args.prompt and len(prompts) < len(keys):
+            ap.error("give a prompt, or a --side-prompt for every side")
+    else:
+        names = [args.agent if args.command in ("probe", "cliff") else args.command]
+    project = args.project.resolve()
+    for signum in (signal.SIGTERM, signal.SIGHUP):
+        signal.signal(signum, _exit_on)
+
+    try:
+        reals = {name: AGENTS[name].discover() for name in dict.fromkeys(names)}
+        journal = Journal(None)
+        if args.command in ("cliff", "swap", "calibrate"):
+            out = args.out or (Path.home() / ".local/state/tare" / args.command
+                               / f"{project.name}-{time.strftime('%Y%m%d-%H%M%S')}")
+            out.mkdir(parents=True, exist_ok=False)
+            journal = Journal(out)
+            if args.command == "cliff":
+                agents = {"a": (AGENTS[args.agent], reals[args.agent], passthrough)}
+                params = {"tails": args.tails, "budget": args.budget, "jobs": args.jobs, "gap_below": args.gap_below}
+            elif args.command == "calibrate":
+                agents = {chr(97 + i): (AGENTS[side[0]], reals[side[0]], side[1:]) for i, side in enumerate(sides)}
+                params = {"runs": args.runs, "jobs": args.jobs, **({"prompts": prompts} if prompts else {})}
+            else:
+                agents = {"a": (AGENTS[args.a], reals[args.a], shlex.split(args.a_args)),
+                          "b": (AGENTS[args.b], reals[args.b], shlex.split(args.b_args))}
+                params = {"cuts": args.cuts, "tails": args.tails, "jobs": args.jobs, "handoff": args.handoff}
+            recipe.write(out, original_argv, args.command, project, args.prompt, args.check, agents, params)
+            print(f"tare {args.command}: run directory {out}", file=sys.stderr)
+            print(f"tare {args.command}: live dashboard {_serve(out, args.port)}", file=sys.stderr)
+        for name in dict.fromkeys(names):
+            agent = AGENTS[name]
+            reading = probe(agent, reals[name], project)
+            print(render(reading, project), file=sys.stderr)
+            journal("probe", agent=name, zero=reading.zero, text=render(reading, project))
+            if args.command == "probe":
+                return 0 if reading.zero else 1
+            if not reading.zero:
+                if not args.allow_dirty:
+                    print("tare: not starting, because the room is not proven clean. Fix what the reading names, "
+                          "or pass --allow-dirty.", file=sys.stderr)
+                    return 1
+                print(f"tare: --allow-dirty given, running {name} in a room whose reading is not tare: 0.00",
+                      file=sys.stderr)
+        agent, real = AGENTS[names[0]], reals[names[0]]
+        if args.command == "cliff":
+            print(cliff(agent, real, project, args.prompt, args.check, out, tails=args.tails, budget=args.budget,
+                        jobs=args.jobs, claude_args=passthrough, journal=journal, gap_below=args.gap_below), end="")
+            print(f"tare cliff: see it again with  tare watch {out}", file=sys.stderr)
+            return 0
+        if args.command == "calibrate":
+            runs = [calibration.Side(key, agent_, real_, list(extra), prompts.get(key))
+                    for key, (agent_, real_, extra) in agents.items()]
+            print(calibration.calibrate(runs, project, args.prompt, args.check, out, runs=args.runs, jobs=args.jobs,
+                                        journal=journal, keep=args.keep), end="")
+            print(f"tare calibrate: see it again with  tare watch {out}", file=sys.stderr)
+            return 0
+        if args.command == "swap":
+            a = Side("a", AGENTS[args.a], reals[args.a], shlex.split(args.a_args), out / "a")
+            b = Side("b", AGENTS[args.b], reals[args.b], shlex.split(args.b_args), out / "b")
+            cuts = [float(c) for c in args.cuts.split(",")]
+            print(swap(a, b, project, args.prompt, args.check, out, cuts=cuts, tails=args.tails, jobs=args.jobs,
+                       workspace_only=args.handoff == "workspace", journal=journal), end="")
+            print(f"tare swap: see it again with  tare watch {out}", file=sys.stderr)
+            return 0
+        flags = agent.room_flags + (agent.yolo if args.yolo else [])
+        with room_home(agent, real) as home:
+            return _run_attached(bwrap(agent, real, home, project, [agent.name, *flags, *passthrough]))
+    except TareError as err:
+        print(f"tare: {err}", file=sys.stderr)
+        return 2
+
+
+def _watch(argv: list[str]) -> int:
+    ap = argparse.ArgumentParser(prog="tare watch", description="the live dashboard of a run, or of several runs on one page")
+    ap.add_argument("dirs", type=Path, nargs="+")
+    ap.add_argument("--port", type=int, default=8777)
+    args = ap.parse_args(argv)
+    missing = [str(d) for d in args.dirs if not (d / "journal.jsonl").exists()]
+    if missing:
+        print(f"tare watch: not a tare run directory (no journal.jsonl): {', '.join(missing)}", file=sys.stderr)
+        return 2
+    dirs = [d.resolve() for d in args.dirs]
+    print(f"tare watch: {_serve(dirs[0] if len(dirs) == 1 else dirs, args.port)}  (Ctrl-C to stop)", file=sys.stderr)
+    try:
+        threading.Event().wait()
+    except KeyboardInterrupt:
+        pass
+    return 0
+
+
+def _judge(command: str, argv: list[str]) -> int:
+    from . import judge as judging
+    ap = argparse.ArgumentParser(prog=f"tare {command}")
+    ap.add_argument("dirs", type=Path, nargs="*" if command == "judge" else "+", default=[Path.cwd()])
+    ap.add_argument("--rubric", type=Path, required=True)
+    ap.add_argument("--threshold", type=float, required=command == "judge")
+    ap.add_argument("--times", type=int, default=10, help="scores per page (judge-noise, default 10)")
+    ap.add_argument("--page", default="index.html", help="the page to render (default index.html)")
+    ap.add_argument("--judge", default="claude --model sonnet", help="the judge agent and its arguments")
+    args = ap.parse_args(argv)
+    name, *extra = shlex.split(args.judge)
+    try:
+        if command == "judge":
+            score, reason = judging.judge(args.dirs[0].resolve(), args.rubric.resolve(), args.page, name, extra)
+            print(f"score {score} (threshold {args.threshold:g}): {reason}")
+            return 0 if score >= args.threshold else 1
+        text, clear = judging.noise([d.resolve() for d in args.dirs], args.rubric.resolve(), args.times,
+                                    args.threshold, args.page, name, extra)
+        print(text, end="")
+        return 0 if clear else 1
+    except (RuntimeError, TareError) as err:
+        print(f"tare {command}: {err}")
+        return 2
+
+
+def _prices(argv: list[str]) -> int:
+    ap = argparse.ArgumentParser(prog="tare prices", description="the price tables for costs that an agent does not "
+                                 f"report. Your own file {usages.config_dir() / 'prices.json'} overrides single models.")
+    ap.add_argument("action", nargs="?", choices=["show", "update"], default="show")
+    ap.add_argument("--url", default=usages.LITELLM_URL, help="where to fetch the table from (default: LiteLLM)")
+    args = ap.parse_args(argv)
+    if args.action == "update":
+        try:
+            print(f"tare prices: {usages.update_prices(args.url)}")
+        except (OSError, ValueError) as err:
+            print(f"tare prices: could not fetch {args.url}: {err}", file=sys.stderr)
+            return 2
+        return 0
+    tables = usages.tables_summary()
+    print("\n".join(f"tare prices: {line}" for line in tables) if tables else
+          "tare prices: no price table yet. Run `tare prices update` to fetch LiteLLM's.")
+    return 0
+
+
+def _rerun(argv: list[str]) -> int:
+    ap = argparse.ArgumentParser(prog="tare rerun", description="repeat a Calibrate, Cliff or Swap run from its recipe")
+    ap.add_argument("dir", type=Path)
+    ap.add_argument("--out", type=Path, help="run directory for the repeat (default: a new one)")
+    args = ap.parse_args(argv)
+    try:
+        r = recipe.read(args.dir)
+    except OSError:
+        print(f"tare rerun: no recipe.json in {args.dir}", file=sys.stderr)
+        return 2
+    reals = {}
+    for agent in r["agents"].values():
+        try:
+            reals[agent["name"]] = AGENTS[agent["name"]].discover()
+        except TareError:
+            pass
+    for note in recipe.drift(r, reals):
+        print(f"tare rerun: note: {note}", file=sys.stderr)
+    spent = args.dir / "usage.json"
+    if spent.exists():
+        before = usages.Usage.from_dict(json.loads(spent.read_text()).get("total"))
+        print(f"tare rerun: the run being repeated used {usages.describe(before)}", file=sys.stderr)
+    command = recipe.with_out(r["argv"], args.out) if args.out else r["argv"]
+    print(f"tare rerun: tare {shlex.join(command)}", file=sys.stderr)
+    return main(command)
+
+
+if __name__ == "__main__":
+    sys.exit(main())