tare-cli 0.2.0__py3-none-any.whl

tare/dashboard.py

@@ -0,0 +1,216 @@
+"""The live dashboard: a local page that shows a Cliff or Swap run as it happens (epic Dashboard).
+
+Everything comes from the run directory: the recipe, the journal, and the live output each
+original run and tail writes. A finished run looks the same as a live one, so `tare watch`
+can show any run directory. Given several, it serves an overview with one row per run, and each
+run's own dashboard under run/<i>/.
+"""
+import json
+import re
+import threading
+import time
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from importlib import resources
+from pathlib import Path
+
+from . import journal as journals
+from . import usage as usages
+from .agents import AGENTS
+from .cliff import wilson
+from .swap import Cut
+
+
+def _activity(agent_name: str | None, stdout: Path) -> str | None:
+    """The latest thing the agent did, from the tail of its live output."""
+    agent = AGENTS.get(agent_name or "")
+    if not agent or not stdout.exists():
+        return None
+    with stdout.open("rb") as f:
+        f.seek(max(0, stdout.stat().st_size - 64 * 1024))
+        lines = f.read().decode(errors="replace").splitlines()
+    for line in reversed(lines):
+        try:
+            described = agent.activity(json.loads(line))
+        except ValueError:
+            continue
+        if described:
+            return described
+    return None
+
+
+def _capsule_count(store: Path) -> int:
+    return max(0, len(list((store / "capsules").glob("*.tar"))) - 1) if store.exists() else 0
+
+
+def state(out: Path) -> dict:
+    events = journals.read(out)
+    recipe = out / "recipe.json"
+    s = {"recipe": json.loads(recipe.read_text()) if recipe.exists() else None, "kind": None, "task": None,
+         "check": None, "project": None, "sides": {}, "params": {}, "phase": "starting", "probes": [], "originals": {},
+         "tails": [], "plan": [], "report": None, "started": events[0]["t"] if events else None,
+         "updated": events[-1]["t"] if events else None, "now": time.time()}
+    tails: dict[str, dict] = {}
+    for e in events:
+        kind = e["event"]
+        if kind == "start":
+            s.update({k: e.get(k) for k in ("kind", "task", "check", "project", "sides")})
+            s["params"] = {k: e.get(k) for k in ("cuts", "tails", "budget")}
+            if s["kind"] == "calibrate":
+                s["originals"] = {k: {"passed": None, "detail": "", "steps": []} for k in s["sides"]}  # no original runs
+        elif kind == "phase":
+            s["phase"] = e["phase"]
+        elif kind == "probe":
+            s["probes"].append({k: e.get(k) for k in ("agent", "zero", "text")})
+        elif kind == "original":
+            s["originals"][e["side"]] = {"passed": e["passed"], "detail": e["detail"], "steps": e["steps"]}
+        elif kind == "plan":
+            s["plan"] = e["cuts"]
+        elif kind == "tail":
+            tail = tails.setdefault(e["id"], {"id": e["id"], "started": e["t"]})
+            tail.update({k: v for k, v in e.items() if k not in ("t", "event")})
+            if e["status"] != "running":
+                tail["ended"] = e["t"]
+        elif kind == "report":
+            s["report"] = e["text"]
+
+    # originals still running: their capsules so far and what the agent does right now
+    for key, side in (s["sides"] or {}).items():
+        if key not in s["originals"]:
+            base = out / side.get("dir", ".")
+            s["originals"][key] = {"running": True, "count": _capsule_count(base / "store"),
+                                   "activity": _activity(side.get("agent"), base / "original" / "stdout.jsonl")}
+    for tail in tails.values():
+        if tail["status"] == "running":
+            tail["activity"] = _activity(tail.get("agent_name"), out / tail["dir"] / "stdout.jsonl")
+    s["tails"] = sorted(tails.values(), key=lambda t: (t["status"] != "running", -t["started"]))
+
+    if s["kind"] == "cliff":
+        s["steps"] = _cliff_steps(tails.values())
+    elif s["kind"] == "swap":
+        s["cells"] = _swap_cells(tails.values(), s["plan"])
+    elif s["kind"] == "calibrate":
+        s["rates"] = _rates(tails.values(), s["sides"])
+    s["usage"] = usages.of_run(events)
+    return s
+
+
+def _cliff_steps(tails) -> list[dict]:
+    steps: dict[int, list] = {}
+    for t in tails:
+        steps.setdefault(t["step"], []).append(t)
+    rows = []
+    for step, ts in sorted(steps.items()):
+        done = [t["status"] == "passed" for t in ts if t["status"] != "running"]
+        lo, hi = wilson(sum(done), len(done))
+        rows.append({"step": step, "passes": sum(done), "n": len(done), "running": sum(t["status"] == "running" for t in ts),
+                     "rate": sum(done) / len(done) if done else None, "lo": lo, "hi": hi})
+    return rows
+
+
+def _rates(tails, sides) -> dict:
+    rates = {}
+    for key in sides:
+        ts = [t for t in tails if t.get("side") == key]
+        done = [t["status"] == "passed" for t in ts if t["status"] != "running"]
+        lo, hi = wilson(sum(done), len(done))
+        # a judge check reports "score N" in its detail line
+        scores = [int(m.group(1)) for t in ts if t["status"] != "running"
+                  and (m := re.search(r"score (\d+)", t.get("detail", "")))]
+        rates[key] = {"passes": sum(done), "n": len(done), "running": sum(t["status"] == "running" for t in ts),
+                      "rate": sum(done) / len(done) if done else None, "lo": lo, "hi": hi, "scores": scores,
+                      "mean": sum(scores) / len(scores) if scores else None,
+                      "times": [t["ended"] - t["started"] for t in ts if "ended" in t]}
+    return rates
+
+
+def _swap_cells(tails, plan) -> list[dict]:
+    cuts = []
+    for i, p in enumerate(plan):
+        cut = Cut(p["fraction"], p["steps"])
+        running: dict[tuple[str, str], int] = {}
+        for t in tails:
+            if t.get("cut") != i:
+                continue
+            native = t["kind"] == "native"
+            key = ("native" if native else t["room"], t["room"] if native else t["agent"])
+            target = cut.native.setdefault(t["room"], []) if native else cut.cells.setdefault(key, [])
+            if t["status"] == "running":
+                running[key] = running.get(key, 0) + 1
+            else:
+                target.append(t["status"] == "passed")
+        complete = all(cut.cells.get((r, m)) for r in "ab" for m in "ab")
+        cuts.append({
+            "fraction": p["fraction"], "steps": p["steps"],
+            "cells": {f"{r}{m}": {"passes": sum(cut.cells.get((r, m), [])), "n": len(cut.cells.get((r, m), [])),
+                                  "running": running.get((r, m), 0)} for r in "ab" for m in "ab"},
+            "native": {r: {"passes": sum(v), "n": len(v), "running": running.get(("native", r), 0)}
+                       for r, v in cut.native.items()},
+            "state": cut.state() if complete else None, "model": cut.model() if complete else None,
+        })
+    return cuts
+
+
+def overview(out: Path) -> dict:
+    """One row of the overview: what a run is and how far it got."""
+    s = state(out)
+    verdict = next((line.strip()[len("verdict"):].strip() for line in (s["report"] or "").splitlines()
+                    if line.strip().startswith("verdict")), None)
+    running = sum(t["status"] == "running" for t in s["tails"])
+    return {"name": out.name, "kind": s["kind"], "task": s["task"], "check": s["check"], "sides": s["sides"],
+            "phase": s["phase"], "running": running, "done": len(s["tails"]) - running, "rates": s.get("rates"),
+            "verdict": verdict}
+
+
+def serve(out: Path | list[Path], port: int = 0) -> tuple[ThreadingHTTPServer, str]:
+    """Serve the dashboard for a run directory, or an overview of several, on 127.0.0.1;
+    returns the server and its URL."""
+    runs = [out] if isinstance(out, Path) else list(out)
+    page = resources.files("tare").joinpath("dashboard.html").read_bytes()
+    index = resources.files("tare").joinpath("overview.html").read_bytes()
+    html = "text/html; charset=utf-8"
+
+    def route(path: str) -> tuple[bytes, str] | str | None:
+        """The body and its type, a path to redirect to, or None."""
+        if len(runs) == 1:
+            run, rest = runs[0], path
+        else:
+            if path in ("/", "/index.html"):
+                return index, html
+            if path == "/runs":
+                return json.dumps([{"i": i, **overview(r)} for i, r in enumerate(runs)]).encode(), "application/json"
+            m = re.fullmatch(r"/run/(\d+)(/.*)?", path)
+            if not m or int(m.group(1)) >= len(runs):
+                return None
+            if not m.group(2):
+                return f"{path}/"
+            run, rest = runs[int(m.group(1))], m.group(2)
+        if rest == "/state":
+            return json.dumps(state(run)).encode(), "application/json"
+        if rest in ("/", "/index.html"):
+            return page, html
+        return None
+
+    class Handler(BaseHTTPRequestHandler):
+        def log_message(self, *args):
+            pass
+
+        def do_GET(self):
+            found = route(self.path.split("?")[0])
+            if found is None or isinstance(found, str):
+                self.send_response(404 if found is None else 301)
+                if found:
+                    self.send_header("location", found)
+                self.send_header("content-length", "0")
+                self.end_headers()
+                return
+            body, kind = found
+            self.send_response(200)
+            self.send_header("content-type", kind)
+            self.send_header("cache-control", "no-store")
+            self.send_header("content-length", str(len(body)))
+            self.end_headers()
+            self.wfile.write(body)
+
+    server = ThreadingHTTPServer(("127.0.0.1", port), Handler)
+    threading.Thread(target=server.serve_forever, daemon=True).start()
+    return server, f"http://localhost:{server.server_address[1]}/"

tare/fake.py

@@ -0,0 +1,183 @@
+"""A fake model endpoint that keeps what the CLI sends.
+
+The CLI under test is pointed at it: Claude Code with ANTHROPIC_BASE_URL (Anthropic
+Messages API), Codex with `-c openai_base_url=...` (OpenAI Responses API), the Antigravity
+CLI with CLOUD_CODE_URL (Google's Cloud Code `v1internal` API). Every request
+body is kept in memory; headers are never stored, so the login token never leaves the
+request. By default each turn is answered with the text "ok", so the CLI ends its loop at
+once; a `respond` policy can script tool calls instead (Anthropic Messages only).
+"""
+import json
+import sys
+import threading
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+
+
+OK = [{"type": "text", "text": "ok"}]
+
+# Cloud Code (Antigravity CLI): the model the fake offers, and the answers to the calls
+# around a turn. The eligibility check needs a tier and a project; a model must be listed
+# to be selectable. Everything else is answered with {}.
+FAKE_MODEL = "tare-fake"
+_GEMINI_OK = {"candidates": [{"content": {"role": "model", "parts": [{"text": "ok"}]}, "finishReason": "STOP"}],
+              "usageMetadata": {"promptTokenCount": 1, "candidatesTokenCount": 1, "totalTokenCount": 2}}
+_CLOUD_CODE = {
+    "loadCodeAssist": {"currentTier": {"id": "free-tier", "name": "Free"}, "cloudaicompanionProject": "tare-fake",
+                       "allowedTiers": [{"id": "free-tier", "name": "Free", "isDefault": True}]},
+    "fetchAvailableModels": {
+        "defaultAgentModelId": FAKE_MODEL,
+        "agentModelSorts": [{"displayName": "Recommended", "groups": [{"modelIds": [FAKE_MODEL]}]}],
+        "models": {FAKE_MODEL: {"displayName": "tare fake", "recommended": True, "model": "MODEL_PLACEHOLDER_M318",
+                                "apiProvider": "API_PROVIDER_GOOGLE_GEMINI", "modelProvider": "MODEL_PROVIDER_GOOGLE",
+                                "maxTokens": 1048576, "maxOutputTokens": 65536}}},
+    "countTokens": {"totalTokens": 1},
+    "generateContent": {"response": _GEMINI_OK},
+}
+
+
+def _stop_reason(blocks: list[dict]) -> str:
+    return "tool_use" if any(b["type"] == "tool_use" for b in blocks) else "end_turn"
+
+
+def _messages_events(model: str, blocks: list[dict]) -> list[tuple[str, dict]]:
+    message = {"id": "msg_tare", "type": "message", "role": "assistant", "model": model, "content": [],
+               "stop_reason": None, "stop_sequence": None, "usage": {"input_tokens": 1, "output_tokens": 1}}
+    events = [("message_start", {"type": "message_start", "message": message})]
+    for i, block in enumerate(blocks):
+        if block["type"] == "text":
+            start, delta = {"type": "text", "text": ""}, {"type": "text_delta", "text": block["text"]}
+        else:
+            start = {"type": "tool_use", "id": block["id"], "name": block["name"], "input": {}}
+            delta = {"type": "input_json_delta", "partial_json": json.dumps(block["input"])}
+        events += [("content_block_start", {"type": "content_block_start", "index": i, "content_block": start}),
+                   ("content_block_delta", {"type": "content_block_delta", "index": i, "delta": delta}),
+                   ("content_block_stop", {"type": "content_block_stop", "index": i})]
+    return events + [
+        ("message_delta", {"type": "message_delta", "delta": {"stop_reason": _stop_reason(blocks), "stop_sequence": None},
+                           "usage": {"output_tokens": 1}}),
+        ("message_stop", {"type": "message_stop"}),
+    ]
+
+
+def _responses_events() -> list[tuple[str, dict]]:
+    item = {"type": "message", "role": "assistant", "id": "msg_tare",
+            "content": [{"type": "output_text", "text": "ok", "annotations": []}]}
+    usage = {"input_tokens": 1, "input_tokens_details": {"cached_tokens": 0}, "output_tokens": 1,
+             "output_tokens_details": {"reasoning_tokens": 0}, "total_tokens": 2}
+    return [
+        ("response.created", {"type": "response.created", "response": {"id": "resp_tare"}}),
+        ("response.output_item.added", {"type": "response.output_item.added", "output_index": 0, "item": item}),
+        ("response.output_text.delta", {"type": "response.output_text.delta", "item_id": "msg_tare",
+                                        "output_index": 0, "content_index": 0, "delta": "ok"}),
+        ("response.output_item.done", {"type": "response.output_item.done", "output_index": 0, "item": item}),
+        ("response.completed", {"type": "response.completed", "response": {"id": "resp_tare", "usage": usage}}),
+    ]
+
+
+class _QuietServer(ThreadingHTTPServer):
+    def handle_error(self, request, client_address):
+        # The CLIs drop connections at will (a refused websocket, the end of a run);
+        # that is not worth a traceback in the user's terminal.
+        if not isinstance(sys.exc_info()[1], ConnectionError):
+            super().handle_error(request, client_address)
+
+
+class Fake:
+    """Context manager: serves on a free port of 127.0.0.1 and collects request bodies."""
+
+    def __init__(self, respond=None):
+        # respond(body) -> content blocks of the answer (text and tool_use), for scripted runs
+        self.respond = respond or (lambda body: OK)
+        self.requests: list[dict] = []
+        self.other_paths: list[str] = []
+        fake = self
+
+        class Handler(BaseHTTPRequestHandler):
+            # HTTP/1.1, so a refused websocket upgrade reads as a plain 404 and Codex
+            # falls back to HTTPS instead of retrying.
+            protocol_version = "HTTP/1.1"
+
+            def log_message(self, *args):
+                pass
+
+            def _json(self, status, payload):
+                data = json.dumps(payload).encode()
+                self.send_response(status)
+                self.send_header("content-type", "application/json")
+                self.send_header("content-length", str(len(data)))
+                self.end_headers()
+                self.wfile.write(data)
+
+            def _not_found(self):
+                fake.other_paths.append(f"{self.command} {self.path.split('?')[0]}")
+                self._json(404, {"type": "error", "error": {"type": "not_found_error", "message": "tare fake"}})
+
+            def _stream(self, events):
+                self.send_response(200)
+                self.send_header("content-type", "text/event-stream")
+                self.send_header("cache-control", "no-cache")
+                self.send_header("connection", "close")
+                self.end_headers()
+                for name, data in events:
+                    self.wfile.write(f"event: {name}\ndata: {json.dumps(data)}\n\n".encode())
+                self.wfile.flush()
+                self.close_connection = True
+
+            def do_GET(self):
+                self._not_found()
+
+            def _body(self) -> bytes:
+                if "chunked" not in (self.headers.get("transfer-encoding") or ""):
+                    return self.rfile.read(int(self.headers.get("content-length") or 0))
+                parts = []  # the Antigravity CLI sends its model requests chunked
+                while size := int(self.rfile.readline().split(b";")[0].strip() or b"0", 16):
+                    parts.append(self.rfile.read(size))
+                    self.rfile.readline()
+                self.rfile.readline()
+                return b"".join(parts)
+
+            def _cloud_code(self, method: str, raw: bytes):
+                if method != "streamGenerateContent":
+                    return self._json(200, _CLOUD_CODE.get(method, {}))
+                fake.requests.append(json.loads(raw))
+                data = f"data: {json.dumps({'response': _GEMINI_OK})}\r\n\r\n".encode()
+                self.send_response(200)
+                self.send_header("content-type", "text/event-stream")
+                self.send_header("content-length", str(len(data)))
+                self.end_headers()
+                self.wfile.write(data)
+
+            def do_POST(self):
+                raw = self._body()
+                path = self.path.split("?")[0]
+                if path.startswith("/v1internal:"):
+                    return self._cloud_code(path.split(":", 1)[1], raw)
+                if path.endswith("/messages/count_tokens"):
+                    return self._json(200, {"input_tokens": 1})
+                if not path.endswith(("/v1/messages", "/responses")):
+                    return self._not_found()
+                if self.headers.get("content-encoding"):
+                    # Codex compresses unless told not to (--disable enable_request_compression)
+                    return self._json(415, {"error": {"message": "tare fake: send uncompressed bodies"}})
+                body = json.loads(raw)
+                fake.requests.append(body)
+                if path.endswith("/responses"):
+                    return self._stream(_responses_events())
+                model = body.get("model", "fake")
+                blocks = fake.respond(body) if body.get("tools") else OK
+                if body.get("stream"):
+                    return self._stream(_messages_events(model, blocks))
+                self._json(200, {"id": "msg_tare", "type": "message", "role": "assistant", "model": model,
+                                 "content": blocks, "stop_reason": _stop_reason(blocks),
+                                 "stop_sequence": None, "usage": {"input_tokens": 1, "output_tokens": 1}})
+
+        self._server = _QuietServer(("127.0.0.1", 0), Handler)
+        self.url = f"http://127.0.0.1:{self._server.server_address[1]}"
+
+    def __enter__(self):
+        threading.Thread(target=self._server.serve_forever, daemon=True).start()
+        return self
+
+    def __exit__(self, *exc):
+        self._server.shutdown()
+        self._server.server_close()

tare/journal.py

@@ -0,0 +1,35 @@
+"""The run journal: everything a Cliff or Swap run did, one JSON line per event.
+
+The dashboard derives its whole state from the run directory, and the journal is its spine:
+phases, probe readings, original runs, tails started and finished, search steps, the report.
+"""
+import json
+import threading
+import time
+from pathlib import Path
+
+
+class Journal:
+    def __init__(self, out: Path | None):
+        self.path = out / "journal.jsonl" if out else None
+        self._lock = threading.Lock()
+
+    def __call__(self, event: str, **data):
+        if not self.path:
+            return
+        line = json.dumps({"t": round(time.time(), 3), "event": event, **data}, ensure_ascii=False)
+        with self._lock, self.path.open("a") as f:
+            f.write(line + "\n")
+
+
+def read(out: Path) -> list[dict]:
+    path = out / "journal.jsonl"
+    if not path.exists():
+        return []
+    events = []
+    for line in path.read_text().splitlines():
+        try:
+            events.append(json.loads(line))
+        except ValueError:
+            continue  # a line being written
+    return events

tare/judge.py

@@ -0,0 +1,108 @@
+"""Judge: a check for work that tests cannot settle, such as the quality of a page (epic Judge).
+
+The finished page is rendered headless in a room of its own (no home directory, the
+workspace read-only) and photographed. A judge agent then gets the screenshot, the source
+and the rubric in a fresh tare room, without any agent or model names, and answers with a
+score. The check passes at or above a threshold. Before a threshold is used, `noise` scores
+the same pages repeatedly: a threshold inside a page's spread would measure the judge's
+dice, not the agent.
+"""
+import json
+import os
+import re
+import shutil
+import statistics
+import subprocess
+import tempfile
+from pathlib import Path
+
+from . import room as rooms
+from .agents import AGENTS, events, strings
+
+VIEWPORT = "1280,900"
+PROMPT = ("You are a strict, careful judge. Read rubric.md. Look at page.png, a screenshot of the page, and at the "
+          "source in the source/ directory. Score the page from 0 to 100 exactly as the rubric says. Reply with "
+          'nothing but one line of JSON: {"score": <0-100>, "reason": "<one sentence>"}')
+
+
+def render(work: Path, page: str, target: Path, wait_ms: int = 2000):
+    """Screenshot of work/page, rendered headless in a room: the workspace read-only, no home directory."""
+    target.parent.mkdir(parents=True, exist_ok=True)
+    shot = Path(tempfile.mkdtemp(prefix="tare-render-"))
+    try:
+        argv = ["bwrap", "--ro-bind", "/usr", "/usr", "--ro-bind", "/etc", "/etc", "--ro-bind", "/opt", "/opt",
+                "--symlink", "usr/bin", "/bin", "--symlink", "usr/lib", "/lib", "--symlink", "usr/lib64", "/lib64",
+                "--proc", "/proc", "--dev", "/dev", "--tmpfs", "/tmp", "--tmpfs", "/home",
+                "--ro-bind", str(work), "/work", "--bind", str(shot), "/shot",
+                "--unshare-user", "--unshare-pid", "--unshare-ipc", "--unshare-uts", "--die-with-parent",
+                "--clearenv", "--setenv", "HOME", "/tmp", "--setenv", "PATH", "/usr/bin:/bin", "--",
+                "google-chrome", "--headless=new", "--no-sandbox", "--disable-gpu", "--hide-scrollbars",
+                f"--window-size={VIEWPORT}", f"--virtual-time-budget={wait_ms}", "--screenshot=/shot/page.png",
+                f"file:///work/{page}"]
+        subprocess.run(argv, capture_output=True, timeout=120, stdin=subprocess.DEVNULL)
+        if not (shot / "page.png").exists():
+            raise RuntimeError(f"the page {page} did not render")
+        shutil.copyfile(shot / "page.png", target)
+    finally:
+        shutil.rmtree(shot, ignore_errors=True)
+
+
+def _score(stdout: str) -> tuple[int, str]:
+    """The last {"score": ...} the judge wrote, in any of its event stream's strings or in plain text."""
+    texts = [s for e in events(stdout) for s in strings(e)] or [stdout]
+    for text in reversed(texts):
+        for match in reversed(list(re.finditer(r'\{[^{}]*"score"\s*:\s*\d+[^{}]*\}', text))):
+            try:
+                data = json.loads(match.group(0))
+                return int(data["score"]), str(data.get("reason", ""))
+            except (ValueError, KeyError):
+                continue
+    raise RuntimeError("the judge gave no score")
+
+
+def judge(work: Path, rubric: Path, page: str = "index.html", agent_name: str = "claude",
+          agent_args: list[str] | None = None, timeout: float = 600) -> tuple[int, str]:
+    """Score the page in work/ by the rubric; the judge agent runs in a fresh, blind room."""
+    agent = AGENTS[agent_name]
+    real = agent.discover()
+    with tempfile.TemporaryDirectory(prefix="tare-judge-") as tmp:
+        bench = Path(tmp)
+        render(work, page, bench / "page.png")
+        shutil.copyfile(rubric, bench / "rubric.md")
+        # the source the agent wrote, without anything that names an agent or its setup
+        shutil.copytree(work, bench / "source", ignore=shutil.ignore_patterns(
+            ".git", ".claude", ".codex", ".pi", "AGENTS.md", "CLAUDE.md", "node_modules"))
+        with rooms.room_home(agent, real) as home:
+            argv = rooms.bwrap(agent, real, home, bench, [agent.name, *agent.room_flags,
+                                                         *agent.run_args(PROMPT, agent_args or [])])
+            proc = subprocess.run(argv, capture_output=True, text=True, timeout=timeout, stdin=subprocess.DEVNULL)
+    _log_usage(agent, proc.stdout)
+    return _score(proc.stdout)
+
+
+def _log_usage(agent, stdout: str):
+    """When tare runs this judge as a check, it names a file for the judge agent's usage."""
+    log = os.environ.get("TARE_USAGE_LOG")
+    if log:
+        with open(log, "a") as f:
+            f.write(json.dumps(agent.usage(stdout).to_dict()) + "\n")
+
+
+def noise(pages: list[Path], rubric: Path, times: int, threshold: float | None, page: str = "index.html",
+          agent_name: str = "claude", agent_args: list[str] | None = None) -> tuple[str, bool]:
+    """Score each page `times` times. Returns the report and whether the threshold sits outside every spread."""
+    lines = [f"tare judge-noise · {agent_name} {' '.join(agent_args or [])}".rstrip(), f"  rubric    {rubric}",
+             "", "  page                              scores                         mean   sd   min-max"]
+    clear = True
+    for p in pages:
+        scores = [judge(p, rubric, page, agent_name, agent_args)[0] for _ in range(times)]
+        mean, sd = statistics.mean(scores), statistics.pstdev(scores)
+        inside = threshold is not None and min(scores) <= threshold <= max(scores)
+        clear = clear and not inside
+        lines.append(f"  {str(p)[-32:]:<32}  {' '.join(f'{x:>3}' for x in scores):<30} {mean:5.1f} {sd:4.1f}  "
+                     f"{min(scores)}-{max(scores)}" + ("  <- threshold inside this page's range" if inside else ""))
+    if threshold is not None:
+        lines += ["", f"  threshold {threshold}: " + ("outside every page's range of scores" if clear else
+                                                     "INSIDE a page's range of scores. There, chance decides pass or "
+                                                     "fail.")]
+    return "\n".join(lines) + "\n", clear

tare/overview.html

@@ -0,0 +1,78 @@
+<!doctype html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<title>tare runs</title>
+<style>
+:root {
+  --bg: #0e1116; --panel: #161b22; --panel-2: #1c222b; --line: #2a313c; --text: #e6edf3; --muted: #8b96a5;
+  --pass: #3fb950; --fail: #f85149; --run: #d29922; --accent: #79c0ff;
+  --mono: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace;
+}
+@media (prefers-color-scheme: light) {
+  :root { --bg: #f6f8fa; --panel: #ffffff; --panel-2: #f0f3f6; --line: #d0d7de; --text: #1f2328; --muted: #59636e;
+          --pass: #1a7f37; --fail: #cf222e; --run: #9a6700; --accent: #0969da; }
+}
+* { box-sizing: border-box; }
+body { margin: 0; background: var(--bg); color: var(--text); font: 14px/1.45 system-ui, -apple-system, "Segoe UI", sans-serif; }
+main { max-width: 1180px; margin: 0 auto; padding: 20px 16px 60px; }
+h1 { font: 600 22px/1 var(--mono); margin: 0 0 16px; letter-spacing: .5px; }
+section.panel { background: var(--panel); border: 1px solid var(--line); border-radius: 10px; padding: 14px 16px; overflow-x: auto; }
+table { width: 100%; border-collapse: collapse; }
+th, td { text-align: left; padding: 8px; border-bottom: 1px solid var(--line); vertical-align: top; }
+th { color: var(--muted); font-weight: 500; font-size: 12px; }
+a { color: var(--accent); text-decoration: none; }
+a:hover { text-decoration: underline; }
+.mono { font-family: var(--mono); font-size: 12.5px; }
+.muted { color: var(--muted); }
+.dot { width: 8px; height: 8px; border-radius: 50%; background: var(--muted); display: inline-block; margin-right: 6px; }
+.dot.pass { background: var(--pass); } .dot.fail { background: var(--fail); } .dot.run { background: var(--run); animation: pulse 1.2s infinite; }
+@keyframes pulse { 50% { opacity: .35; } }
+.task { max-width: 340px; overflow-wrap: anywhere; }
+</style>
+</head>
+<body>
+<main>
+  <h1>tare runs</h1>
+  <section class="panel"><table>
+    <thead><tr><th>run</th><th>phase</th><th>task</th><th>sides</th><th>progress</th><th>result</th></tr></thead>
+    <tbody id="rows"><tr><td colspan="6" class="muted">loading…</td></tr></tbody>
+  </table></section>
+</main>
+<script>
+const esc = (v) => String(v ?? "").replace(/[&<>"']/g, (c) => ({ "&": "&amp;", "<": "&lt;", ">": "&gt;", '"': "&quot;", "'": "&#39;" }[c]));
+const short = (t, n) => (t && t.length > n ? t.slice(0, n - 1) + "…" : t || "");
+
+function result(r) {
+  if (r.verdict) return esc(r.verdict);
+  if (!r.rates) return `<span class="muted">–</span>`;
+  return Object.entries(r.rates).map(([k, x]) => {
+    const score = x.mean != null ? ` · mean ${x.mean.toFixed(1)}` : "";
+    return `<div class="mono">${esc(k)} ${x.passes}/${x.n} ${r.check ? "passed" : "finished"}${score}</div>`;
+  }).join("");
+}
+
+function row(r) {
+  const phase = r.phase === "done" ? "pass" : "run";
+  const sides = Object.entries(r.sides || {}).map(([k, s]) =>
+    `<div class="mono">${esc(k)} ${esc(s.agent)} ${esc((s.args || []).join(" "))}${s.prompt ? `<div class="muted">${esc(short(s.prompt, 60))}</div>` : ""}</div>`).join("");
+  return `<tr><td><a href="run/${r.i}/"><b>${esc(r.kind || "run")}</b></a><div class="muted mono">${esc(r.name)}</div></td>
+    <td><span class="dot ${phase}"></span>${esc(r.phase)}</td>
+    <td class="task">${esc(short(r.task, 140)) || `<span class="muted">a prompt per side</span>`}</td>
+    <td>${sides}</td><td class="mono">${r.done} ${r.kind === "calibrate" ? "runs" : "tails"} done${r.running ? ` · ${r.running} running` : ""}</td><td>${result(r)}</td></tr>`;
+}
+
+async function tick() {
+  try {
+    const runs = await (await fetch("runs", { cache: "no-store" })).json();
+    document.getElementById("rows").innerHTML = runs.map(row).join("");
+  } catch (e) {
+    document.getElementById("rows").innerHTML = `<tr><td colspan="6" class="muted">disconnected</td></tr>`;
+  }
+  setTimeout(tick, 2000);
+}
+tick();
+</script>
+</body>
+</html>