agent-testbench 0.1.0__py3-none-any.whl

agent_testbench/__init__.py

@@ -0,0 +1,3 @@
+"""agent-testbench: a test bench your coding agent operates - live GPU sessions and verified runs on Modal,
+Colab or this machine, inside a budget."""
+__version__ = "0.1.0"

agent_testbench/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+main()

agent_testbench/backends/__init__.py

@@ -0,0 +1,59 @@
+"""Where a plan runs. Each backend takes a launched run and returns (report, cost in US dollars)."""
+from __future__ import annotations
+
+import fnmatch
+import shutil
+from pathlib import Path
+
+
+def get_backend(name: str):
+    if name == "local":
+        from .local import LocalBackend
+        return LocalBackend
+    if name == "modal":
+        from .modal_backend import ModalBackend
+        return ModalBackend
+    if name == "colab":
+        from .colab_backend import ColabBackend
+        return ColabBackend
+    raise ValueError(f"unknown backend {name!r}")
+
+
+def cancel_remote(state: dict) -> str:
+    """Cancel remote work for a run whose driver is gone, from the handles saved in state.json."""
+    if state.get("modal_call_id"):
+        from .modal_backend import cancel_call
+        return cancel_call(state["modal_call_id"])
+    if state.get("colab_session"):
+        from .colab_backend import stop_session
+        return stop_session(state["colab_session"], state.get("colab_cli_args", []))
+    return ""
+
+
+def excluded(rel: str, patterns: list[str]) -> bool:
+    parts = Path(rel).parts
+    return any(fnmatch.fnmatch(rel, p) or any(fnmatch.fnmatch(part, p) for part in parts) for p in patterns)
+
+
+def copy_project(root: Path, dest: Path, exclude: list[str]) -> Path:
+    """A scratch copy of the project, without the excluded files (and never .testbench itself)."""
+    root, dest = Path(root), Path(dest)
+    patterns = list(exclude) + [".testbench"]
+
+    def ignore(directory, names):
+        rel_dir = Path(directory).relative_to(root)
+        return [n for n in names if excluded(str(rel_dir / n) if str(rel_dir) != "." else n, patterns)]
+
+    shutil.copytree(root, dest, ignore=ignore, dirs_exist_ok=True, symlinks=True)
+    return dest
+
+
+def project_files(root: Path, exclude: list[str]) -> list[Path]:
+    root = Path(root)
+    patterns = list(exclude) + [".testbench"]
+    out = []
+    for p in sorted(root.rglob("*")):
+        rel = str(p.relative_to(root))
+        if p.is_file() and not excluded(rel, patterns):
+            out.append(p)
+    return out

agent_testbench/backends/colab_backend.py

@@ -0,0 +1,157 @@
+"""Run on a Colab VM through Google's Colab CLI (`uv tool install google-colab-cli`).
+
+Per run: allocate a session (CPU or the plan's GPU), install the pinned basics, upload the project as one
+bundle (in chunks - uploads over ~100 MB fail), execute the plan with a timeout equal to its budget, bring
+the run folder back, and always stop the session, so compute units stop with it. The project lands in
+/content, where Colab notebooks expect their files.
+
+Colab notes: VMs can be reclaimed after a couple of hours even while busy, so keep Colab plans short and
+use Modal for long jobs; set `colab: {auth: adc}` in testbench.yaml if you log in with Application Default
+Credentials.
+"""
+from __future__ import annotations
+
+import io
+import json
+import shutil
+import subprocess
+import tarfile
+import tempfile
+import time
+from pathlib import Path
+
+from .. import budget
+from . import project_files
+
+CHUNK = 45 * 1024 * 1024
+PKG = Path(__file__).resolve().parents[1]          # the agent_testbench package, shipped with the project
+
+BOOTSTRAP = r'''
+import glob, json, os, subprocess, sys, tarfile
+parts = sorted(glob.glob("/content/.testbench_bundle.part*"))
+with open("/content/.testbench_bundle.tar.gz", "wb") as out:
+    for p in parts:
+        with open(p, "rb") as f:
+            out.write(f.read())
+        os.remove(p)
+safe = {"filter": "data"} if hasattr(tarfile, "data_filter") else {}
+with tarfile.open("/content/.testbench_bundle.tar.gz") as t:
+    for m in t.getmembers():
+        if m.name.startswith("project/"):
+            m.name = m.name[len("project/"):]
+            if m.name:
+                t.extract(m, "/content", **safe)
+        elif m.name.startswith("pkg/"):
+            m.name = m.name[len("pkg/"):]
+            t.extract(m, "/content/.testbench_pkg", **safe)
+sys.path.insert(0, "/content/.testbench_pkg")
+os.environ["TESTBENCH_CACHE"] = "/content/.testbench_cache"
+os.makedirs("/content/.testbench_cache", exist_ok=True)
+from agent_testbench.executor import execute_plan
+run_dir = "/content/.testbench_run/" + RUN_ID
+report = execute_plan(PLAN, project_dir="/content", run_dir=run_dir, workdir="/content", meta=META)
+print("TESTBENCH_DONE " + json.dumps({"status": report["status"], "minutes": report.get("minutes")}), flush=True)
+'''
+
+PACK = r'''
+import os, tarfile
+d = "/content/.testbench_run/" + RUN_ID
+with tarfile.open("/content/.testbench_result.tar.gz", "w:gz") as t:
+    if os.path.isdir(d):
+        t.add(d, arcname=".")
+print("TESTBENCH_PACKED", os.path.getsize("/content/.testbench_result.tar.gz"), flush=True)
+'''
+
+
+def cli_args(cfg: dict) -> list[str]:
+    c = cfg.get("colab") or {}
+    return [c.get("cli", "colab")] + (["--auth", c["auth"]] if c.get("auth") else [])
+
+
+def stop_session(session: str, args: list[str]) -> str:
+    try:
+        subprocess.run([*(args or ["colab"]), "stop", "-s", session], capture_output=True, text=True, timeout=120)
+        return f"Colab session {session} stopped"
+    except Exception as exc:
+        return f"could not stop Colab session {session}: {exc} - stop it with `colab stop -s {session}`"
+
+
+class ColabBackend:
+    def __init__(self, root: Path, cfg: dict, run_dir: Path, plan: dict, meta: dict):
+        self.root, self.cfg, self.run_dir, self.plan, self.meta = Path(root), cfg, Path(run_dir), plan, meta
+        self.args = cli_args(cfg)
+        self.session = ("testbench-" + run_dir.name)[:60]
+        self.started = None
+
+    def _colab(self, *argv, timeout: float = 900, check: bool = True, input: str | None = None):
+        cmd = [*self.args, *argv]
+        proc = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout, input=input)
+        with (self.run_dir / "colab.log").open("a") as log:
+            log.write(f"$ {' '.join(cmd[:4])} ...\n{proc.stdout[-4000:]}{proc.stderr[-4000:]}\n")
+        if check and proc.returncode != 0:
+            raise RuntimeError(f"`colab {argv[0]}` failed (exit {proc.returncode}): "
+                               f"{(proc.stderr or proc.stdout).strip().splitlines()[-1:] or ''}. See colab.log in the run folder.")
+        return proc
+
+    def _bundle(self) -> Path:
+        path = Path(tempfile.mkdtemp()) / "bundle.tar.gz"
+        with tarfile.open(path, "w:gz") as t:
+            for f in project_files(self.root, self.cfg["exclude"]):
+                t.add(f, arcname="project/" + str(f.relative_to(self.root)))
+            t.add(PKG, arcname="pkg/agent_testbench", filter=lambda m: None if "__pycache__" in m.name else m)
+        return path
+
+    def run(self):
+        from .. import runs, report as R
+        if not shutil.which(self.args[0]):
+            raise RuntimeError("The Colab CLI is not installed: `uv tool install google-colab-cli`, then log in once "
+                               "with `colab new` (or set colab.auth: adc in testbench.yaml).")
+        gpu = (self.plan.get("gpu") or "").upper()
+        runs.write_state(self.run_dir, colab_session=self.session, colab_cli_args=self.args)
+        print(f"[testbench] allocating Colab session {self.session} ({gpu or 'CPU'})", flush=True)
+        self._colab("new", "-s", self.session, *(["--gpu", gpu] if gpu else []), timeout=900)
+        self.started = time.time()
+        try:
+            pkgs = ["nbformat>=5.9", "nbclient>=0.10", "ipykernel>=6.29", "pyyaml>=6", *self.cfg["colab"].get("pip", [])]
+            self._colab("install", "-s", self.session, *pkgs, timeout=1200)
+            if self.cfg["colab"].get("requirements"):
+                self._colab("install", "-s", self.session, "-r", str(self.root / self.cfg["colab"]["requirements"]), timeout=1800)
+            bundle = self._bundle()
+            data = bundle.read_bytes()
+            for i in range(0, max(1, len(data)), CHUNK):
+                part = bundle.with_name(f"part{i // CHUNK:03d}")
+                part.write_bytes(data[i:i + CHUNK])
+                self._colab("upload", "-s", self.session, str(part), f"/content/.testbench_bundle.part{i // CHUNK:03d}", timeout=900)
+            minutes = sum(s["max_minutes"] for s in self.plan["steps"])
+            script = (f"RUN_ID = {self.run_dir.name!r}\nPLAN = json.loads({json.dumps(self.plan)!r})\n"
+                      f"META = json.loads({json.dumps(self.meta)!r})\n")
+            boot = Path(tempfile.mkdtemp()) / "testbench_boot.py"
+            boot.write_text("import json\n" + script + BOOTSTRAP)
+            print(f"[testbench] running the plan on Colab (up to {minutes:.0f} min)", flush=True)
+            proc = self._colab("exec", "-s", self.session, "-f", str(boot), "--timeout", str(int(minutes * 60 + 120)),
+                               timeout=minutes * 60 + 600, check=False)
+            (self.run_dir / "colab_exec.log").write_text(proc.stdout + proc.stderr)
+            pack = Path(tempfile.mkdtemp()) / "testbench_pack.py"
+            pack.write_text(f"RUN_ID = {self.run_dir.name!r}\n" + PACK)
+            self._colab("exec", "-s", self.session, "-f", str(pack), "--timeout", "300", timeout=600)
+            local = Path(tempfile.mkdtemp()) / "result.tar.gz"
+            self._colab("download", "-s", self.session, "/content/.testbench_result.tar.gz", str(local), timeout=1800)
+            with tarfile.open(local) as t:
+                t.extractall(self.run_dir, **({"filter": "data"} if hasattr(tarfile, "data_filter") else {}))
+        finally:
+            print(f"[testbench] {stop_session(self.session, self.args)}", flush=True)
+        rep = R.load(self.run_dir)
+        if not rep:
+            log = (self.run_dir / "colab_exec.log")
+            tail = "\n".join(log.read_text(errors="replace").strip().splitlines()[-20:]) if log.exists() else ""
+            rep = {**self.meta, "steps": [], "started": self.started, "status": "error",
+                   "stopped_because": "the plan never started on the Colab VM",
+                   "launch_error": "The VM could not start the plan. Its last output:\n\n```\n" + tail + "\n```"}
+        return rep, self.cost(rep)
+
+    def cancel(self) -> str:
+        return stop_session(self.session, self.args)
+
+    def cost(self, rep: dict) -> float:
+        minutes = (time.time() - self.started) / 60 if self.started else 0.0
+        return budget.rate_per_minute(self.cfg, self.plan) * minutes

agent_testbench/backends/local.py

@@ -0,0 +1,25 @@
+"""Run on this machine: free, immediate, and the first rung of every ladder."""
+from __future__ import annotations
+
+from pathlib import Path
+
+from .. import executor
+from . import copy_project
+
+
+class LocalBackend:
+    def __init__(self, root: Path, cfg: dict, run_dir: Path, plan: dict, meta: dict):
+        self.root, self.cfg, self.run_dir, self.plan, self.meta = Path(root), cfg, Path(run_dir), plan, meta
+
+    def run(self):
+        workdir = self.root
+        if self.plan.get("workdir") == "copy":
+            workdir = copy_project(self.root, self.run_dir / "work", self.cfg["exclude"])
+        rep = executor.execute_plan(self.plan, project_dir=self.root, run_dir=self.run_dir, workdir=workdir, meta=self.meta)
+        return rep, 0.0
+
+    def cancel(self) -> str:
+        return ""
+
+    def cost(self, rep: dict) -> float:
+        return 0.0

agent_testbench/backends/modal_backend.py

@@ -0,0 +1,212 @@
+"""Run on Modal: one deployed app per project, one function per plan, results on a Modal volume.
+
+The app is generated from testbench.yaml into .testbench/modal_app.py (readable, not hand-edited), deployed, and
+the plan's function is spawned, so the run keeps going if this machine sleeps or loses its network. Lessons
+built in:
+  - the function's timeout is the plan's budget, so the worst case holds on Modal's side;
+  - CPU and memory are always explicit (a GPU function without cpu= gets one core, and the GPU waits on it);
+  - no retries and a 2-second scaledown, so a finished or failed run stops billing at once;
+  - local files are added as the image's last steps, so editing code does not rebuild the image.
+"""
+from __future__ import annotations
+
+import json
+import subprocess
+import sys
+import textwrap
+import threading
+import time
+from pathlib import Path
+
+from .. import budget
+
+APP_TEMPLATE = '''\
+# Generated by agent-testbench from testbench.yaml - edit testbench.yaml, not this file. Regenerated on every Modal run.
+import json
+import threading
+import time
+from pathlib import Path
+
+import modal
+
+app = modal.App({app_name!r})
+volume = modal.Volume.from_name({volume_name!r}, create_if_missing=True)
+image = (
+    modal.Image.debian_slim(python_version={python!r})
+{image_steps}
+    .add_local_python_source("agent_testbench")
+    .add_local_dir({root!r}, "/project", ignore={ignore!r})
+)
+
+
+def _run(run_id: str, plan: dict, meta: dict) -> dict:
+    import os
+    import shutil
+    from agent_testbench.executor import execute_plan
+
+    work = Path("/work/project")
+    shutil.copytree("/project", work, dirs_exist_ok=True, symlinks=True)
+    if not Path("/content").exists():          # Colab notebooks expect to live in /content
+        Path("/content").symlink_to(work)
+    os.environ["TESTBENCH_CACHE"] = "/testbench/cache"
+    Path("/testbench/cache").mkdir(parents=True, exist_ok=True)
+    run_dir = Path("/testbench/runs") / run_id
+    stop = threading.Event()
+
+    def keep_committing():                      # live logs reach the volume about once a minute
+        while not stop.wait(60):
+            try:
+                volume.commit()
+            except Exception:
+                pass
+
+    threading.Thread(target=keep_committing, daemon=True).start()
+    try:
+        report = execute_plan(plan, project_dir=work, run_dir=run_dir, workdir=work, meta=meta, on_progress=volume.commit)
+    finally:
+        stop.set()
+        volume.commit()
+    return {{"status": report["status"], "minutes": report.get("minutes")}}
+{functions}
+'''
+
+FUNCTION_TEMPLATE = '''
+
+@app.function(image=image, gpu={gpu!r}, cpu={cpu!r}, memory={memory!r}, timeout={timeout!r}, retries=0,
+              scaledown_window=2, volumes={{"/testbench": volume}}, secrets=[{secrets}])
+def {fn}(run_id: str, plan: dict, meta: dict) -> dict:
+    return _run(run_id, plan, meta)
+'''
+
+
+def app_name(cfg: dict) -> str:
+    return f"testbench-{cfg['project']}"
+
+
+def fn_name(plan_name: str) -> str:
+    return "plan_" + "".join(c if c.isalnum() else "_" for c in plan_name)
+
+
+def generate(cfg: dict) -> str:
+    m = cfg["modal"]
+    steps = []
+    if m.get("apt"):
+        steps.append(f"    .apt_install({', '.join(repr(x) for x in m['apt'])})")
+    base = ["nbformat>=5.9", "nbclient>=0.10", "ipykernel>=6.29", "pyyaml>=6"]
+    uses_web = any(s["kind"] == "web" for p in cfg["plans"].values() if p["backend"] == "modal" for s in p["steps"])
+    if uses_web:
+        base.append("playwright>=1.45")
+    steps.append(f"    .pip_install({', '.join(repr(x) for x in base + list(m.get('pip') or []))})")
+    if m.get("requirements"):
+        req = Path(cfg["root"]) / m["requirements"]
+        steps.append(f"    .pip_install_from_requirements({str(req)!r})")
+    if uses_web:
+        steps.append('    .run_commands("playwright install --with-deps chromium")')
+    for cmd in m.get("run_commands") or []:
+        steps.append(f"    .run_commands({cmd!r})")
+    ignore = []
+    for p in list(cfg["exclude"]) + [".testbench"]:
+        ignore += [p] if "/" in p else [p, f"**/{p}"]
+    functions = []
+    for name, plan in cfg["plans"].items():
+        if plan["backend"] != "modal":
+            continue
+        minutes = sum(s["max_minutes"] for s in plan["steps"])
+        secrets = sorted({s for st in plan["steps"] for s in st["secrets"]})
+        functions.append(FUNCTION_TEMPLATE.format(
+            gpu=plan.get("gpu"), cpu=float(plan.get("cpu", m["cpu"])), memory=int(float(plan.get("memory_gb", m["memory_gb"])) * 1024),
+            timeout=int((minutes + 3) * 60), fn=fn_name(name),
+            secrets=", ".join(f"modal.Secret.from_name({s!r})" for s in secrets)))
+    return APP_TEMPLATE.format(app_name=app_name(cfg), volume_name=app_name(cfg), python=str(m["python"]),
+                               image_steps="\n".join(steps), root=str(cfg["root"]), ignore=ignore,
+                               functions="".join(functions))
+
+
+def cancel_call(call_id: str) -> str:
+    import modal
+    try:
+        modal.FunctionCall.from_id(call_id).cancel(terminate_containers=True)
+        return f"Modal call {call_id} cancelled"
+    except Exception as exc:
+        return f"could not cancel Modal call {call_id}: {exc}"
+
+
+class ModalBackend:
+    def __init__(self, root: Path, cfg: dict, run_dir: Path, plan: dict, meta: dict):
+        self.root, self.cfg, self.run_dir, self.plan, self.meta = Path(root), cfg, Path(run_dir), plan, meta
+        self.call = None
+        self.remote_started = None
+        self._seen: dict[str, tuple] = {}
+
+    def _log(self, text: str) -> None:
+        print(f"[testbench] {text}", flush=True)
+
+    def deploy(self) -> None:
+        app_file = self.root / ".testbench" / "modal_app.py"
+        app_file.parent.mkdir(parents=True, exist_ok=True)
+        app_file.write_text(generate(self.cfg))
+        self._log(f"deploying {app_name(self.cfg)} from {app_file}")
+        proc = subprocess.run([sys.executable, "-m", "modal", "deploy", str(app_file)], cwd=self.root,
+                              capture_output=True, text=True, timeout=1800)
+        (self.run_dir / "modal_deploy.log").write_text(proc.stdout + proc.stderr)
+        if proc.returncode != 0:
+            tail = "\n".join((proc.stdout + proc.stderr).strip().splitlines()[-25:])
+            raise RuntimeError(f"`modal deploy` failed (is Modal set up? run `modal setup`):\n{tail}")
+
+    def run(self):
+        import modal
+        from .. import runs
+        self.deploy()
+        fn = modal.Function.from_name(app_name(self.cfg), fn_name(self.meta["plan"]))
+        self.call = fn.spawn(self.run_dir.name, self.plan, self.meta)
+        self.remote_started = time.time()
+        runs.write_state(self.run_dir, modal_call_id=self.call.object_id, modal_app=app_name(self.cfg))
+        self._log(f"spawned Modal call {self.call.object_id}")
+        volume = modal.Volume.from_name(app_name(self.cfg))
+        result = None
+        while True:
+            try:
+                result = self.call.get(timeout=20)
+                break
+            except (TimeoutError, modal.exception.TimeoutError):
+                self._mirror(volume, light=True)
+            except modal.exception.FunctionTimeoutError:
+                result = {"status": "timeout"}
+                break
+        self._mirror(volume, light=False)
+        from .. import report as R
+        rep = R.load(self.run_dir) or {**self.meta, "steps": [], "started": self.remote_started, "status": "error"}
+        if result and result.get("status") == "timeout":
+            rep.update(status="failed", stopped_because="the plan ran past its total max_minutes on Modal")
+        return rep, self.cost(rep)
+
+    def _mirror(self, volume, light: bool) -> None:
+        """Copy the run's files from the volume: progress and logs while it runs, everything at the end."""
+        prefix = f"runs/{self.run_dir.name}"
+        try:
+            entries = volume.listdir(prefix, recursive=True)
+        except Exception:
+            return
+        for e in entries:
+            if getattr(e.type, "name", str(e.type)) != "FILE":
+                continue
+            rel = Path(e.path).relative_to(prefix)
+            if light and not (rel.name in ("progress.jsonl", "report.json", "log.txt", "result.json")):
+                continue
+            stamp = (getattr(e, "size", None), getattr(e, "mtime", None))
+            if self._seen.get(e.path) == stamp and stamp != (None, None):
+                continue                                 # unchanged since the last copy
+            self._seen[e.path] = stamp
+            target = self.run_dir / rel
+            target.parent.mkdir(parents=True, exist_ok=True)
+            try:
+                target.write_bytes(b"".join(volume.read_file(e.path)))
+            except Exception:
+                pass
+
+    def cancel(self) -> str:
+        return cancel_call(self.call.object_id) if self.call else ""
+
+    def cost(self, rep: dict) -> float:
+        minutes = (time.time() - self.remote_started) / 60 if self.remote_started else 0.0
+        return budget.rate_per_minute(self.cfg, self.plan) * (minutes + 0.5)

agent_testbench/budget.py

@@ -0,0 +1,113 @@
+"""Money: what a plan can cost at worst, what runs have cost, and whether a launch is allowed.
+
+The worst case of a plan is its hardware's rate times the sum of its steps' max_minutes. Remote steps are
+also stopped on the remote side at that limit (a Modal function timeout, a Colab exec timeout), so the
+estimate is a bound, not a hope - it holds even if this machine goes offline mid-run.
+"""
+from __future__ import annotations
+
+import datetime as dt
+import json
+from pathlib import Path
+
+from . import config as C
+
+# Modal, US dollars per second (modal.com/pricing, checked 2026-10). Override any of them under `pricing:`.
+MODAL_GPU_PER_S = {"B300": 0.001972, "B200": 0.001736, "H200": 0.001261, "H100": 0.001097, "RTX-PRO-6000": 0.000842,
+                   "A100-80GB": 0.000694, "A100-40GB": 0.000583, "A100": 0.000583, "L40S": 0.000542, "A10": 0.000306,
+                   "A10G": 0.000306, "L4": 0.000222, "T4": 0.000164}
+MODAL_CPU_CORE_PER_S = 0.0000131
+MODAL_MEM_GIB_PER_S = 0.00000222
+# Colab bills compute units. These per-hour rates are approximate and vary by account and region; `colab usage`
+# shows your actual rate - put it under `pricing: colab_units_per_hour:`. Pay-as-you-go is ~$10 per 100 units.
+COLAB_UNITS_PER_HOUR = {"CPU": 0.1, "T4": 1.8, "L4": 2.0, "G4": 4.0, "A100": 5.4, "H100": 9.0}
+COLAB_USD_PER_UNIT = 0.10
+
+
+def rate_per_minute(cfg: dict, plan: dict) -> float:
+    """US dollars per minute for the plan's hardware; 0 for local plans."""
+    prices = cfg.get("pricing") or {}
+    backend = plan["backend"]
+    if backend == "local":
+        return 0.0
+    gpu = (plan.get("gpu") or "").split(":")[0].upper()
+    count = int(plan["gpu"].split(":")[1]) if plan.get("gpu") and ":" in plan["gpu"] else 1
+    if backend == "modal":
+        gpu_rates = {**MODAL_GPU_PER_S, **(prices.get("modal_gpu_per_s") or {})}
+        cpu = float(plan.get("cpu") or cfg["modal"]["cpu"])
+        mem = float(plan.get("memory_gb") or cfg["modal"]["memory_gb"])
+        per_s = (gpu_rates.get(gpu, 0.0) * count if gpu else 0.0) \
+            + cpu * float(prices.get("modal_cpu_core_per_s", MODAL_CPU_CORE_PER_S)) \
+            + mem * float(prices.get("modal_mem_gib_per_s", MODAL_MEM_GIB_PER_S))
+        return per_s * 60
+    units = {**COLAB_UNITS_PER_HOUR, **(prices.get("colab_units_per_hour") or {})}
+    return units.get(gpu or "CPU", 0.0) * float(prices.get("colab_usd_per_unit", COLAB_USD_PER_UNIT)) / 60
+
+
+def estimate(cfg: dict, hardware: dict, minutes: float, label: str) -> dict:
+    """Worst case for `minutes` on `hardware` ({backend, gpu, cpu?, memory_gb?}); a remote machine also pays for
+    starting up and installing, so a few minutes are added."""
+    rate = rate_per_minute(cfg, hardware)
+    overhead = 0 if hardware["backend"] == "local" else 5
+    return {"what": label, "backend": hardware["backend"], "gpu": hardware.get("gpu"), "usd_per_minute": rate,
+            "max_minutes": float(minutes), "worst_case_usd": round(rate * (float(minutes) + overhead), 4)}
+
+
+def worst_case(cfg: dict, plan_name: str) -> dict:
+    plan = cfg["plans"][plan_name]
+    return {**estimate(cfg, plan, C.plan_minutes(plan), f"plan {plan_name}"), "plan": plan_name}
+
+
+def ledger_path(root: Path) -> Path:
+    return Path(root) / ".testbench" / "ledger.jsonl"
+
+
+def record(root: Path, entry: dict) -> None:
+    p = ledger_path(root)
+    p.parent.mkdir(parents=True, exist_ok=True)
+    with p.open("a") as f:
+        f.write(json.dumps({"time": dt.datetime.now().isoformat(timespec="seconds"), **entry}) + "\n")
+
+
+def entries(root: Path) -> list[dict]:
+    p = ledger_path(root)
+    if not p.exists():
+        return []
+    return [json.loads(line) for line in p.read_text().splitlines() if line.strip()]
+
+
+def spent_today(root: Path) -> float:
+    """Dollars committed today: finished runs at their measured cost, unfinished ones at their worst case."""
+    today = dt.date.today().isoformat()
+    runs: dict[str, float] = {}
+    for e in entries(root):
+        if not e["time"].startswith(today):
+            continue
+        if e["event"] == "launch":
+            runs[e["run"]] = e["worst_case_usd"]
+        elif e["event"] == "finish":
+            runs[e["run"]] = e["usd"]
+    return round(sum(runs.values()), 4)
+
+
+def check(cfg: dict, est: dict, approve: bool, how_to_lower: str) -> tuple[bool, str, dict]:
+    """(allowed, message, estimate). Over max_run_usd or the day's budget: refused. Over ask_above_usd: needs
+    approve=True, which an agent may only pass after a person said yes."""
+    b = cfg["budget"]
+    cost, what = est["worst_case_usd"], est["what"]
+    if cost > b["max_run_usd"]:
+        return False, (f"Refused: {what} could cost up to ${cost:.2f} ({est['max_minutes']:.0f} min at "
+                       f"${est['usd_per_minute']:.4f}/min), over budget.max_run_usd ${b['max_run_usd']:.2f}. "
+                       f"{how_to_lower}, use a cheaper GPU, or have a person raise the budget."), est
+    today = spent_today(Path(cfg["root"]))
+    if today + cost > b["max_day_usd"]:
+        return False, (f"Refused: ${today:.2f} already committed today; {what} could add ${cost:.2f}, over "
+                       f"budget.max_day_usd ${b['max_day_usd']:.2f}."), est
+    if cost > b["ask_above_usd"] and not approve:
+        return False, (f"Needs approval: {what} could cost up to ${cost:.2f} (above budget.ask_above_usd "
+                       f"${b['ask_above_usd']:.2f}). Ask a person; if they agree, rerun with --approve."), est
+    return True, f"Within budget: up to ${cost:.2f} (today so far ${today:.2f} of ${b['max_day_usd']:.2f}).", est
+
+
+def check_launch(cfg: dict, plan_name: str, approve: bool) -> tuple[bool, str, dict]:
+    return check(cfg, worst_case(cfg, plan_name), approve, "Lower the steps' max_minutes")