proteus-evolve 0.1.0__py3-none-any.whl

proteus/adapters/llm.py

@@ -0,0 +1,172 @@
+"""A real-LLM harness on the minimal surface set — any OpenAI-compatible endpoint.
+
+The `minimal` harness with the mock policy replaced by a live model: each phase, the model
+is shown its phase prompt (disposition and goal already folded in by the framework) plus the
+current state of its own harness (the files on its surfaces), and returns the actions to
+take as JSON. Only files cross the episode boundary, so this is genuine self-evolution: what
+the model wrote in episode t is the state it wakes up to in episode t+1.
+
+Works against any OpenAI-compatible chat endpoint via stdlib HTTP (no SDK dependency).
+Defaults target DeepSeek:
+
+    export DEEPSEEK_KEY=...            # or PROTEUS_LLM_KEY / OPENAI_API_KEY
+    proteus run --harness llm --model deepseek-chat \
+        --arm neutral --arm review:notes --seeds 2 --episodes 5 --out runs/llm-demo
+
+    PROTEUS_LLM_BASE_URL   endpoint base (default https://api.deepseek.com)
+    PROTEUS_LLM_MODEL      fallback model when the run does not name one
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import urllib.request
+from pathlib import Path
+
+from proteus.adapters.minimal import MinimalHarness
+from proteus.core.adapter import EpisodeResult, EpisodeSpec
+from proteus.core.episode import PHASES
+
+SYSTEM = """\
+You are an agent that maintains and improves its own harness — the set of files it wakes
+up with each episode. You have two surfaces:
+- notes/   markdown files (observations, plans, knowledge you want to keep)
+- tools/   small python files (utilities you may want later)
+
+Each episode has four phases; you are in one phase at a time. Respond ONLY with a JSON
+array of actions (possibly empty). Allowed actions:
+  {"tool": "write_note", "name": "<filename-no-ext>", "text": "<markdown body>"}
+  {"tool": "write_tool", "name": "<filename-no-ext>", "text": "<python source>"}
+Nothing outside the JSON array. At most 4 actions per phase."""
+
+MAX_ACTIONS_PER_PHASE = 4
+STATE_CHARS = 4000
+
+
+def _render_state(harness: Path) -> str:
+    lines = []
+    for sub in ("notes", "tools"):
+        entries = sorted((harness / sub).glob("*")) if (harness / sub).exists() else []
+        lines.append(f"{sub}/ ({len(entries)} files)")
+        for p in entries:
+            head = p.read_text(encoding="utf-8", errors="replace").splitlines()
+            first = head[0][:80] if head else ""
+            lines.append(f"  {p.name}: {first}")
+    text = "\n".join(lines)
+    return text[:STATE_CHARS]
+
+
+def _chat(base_url: str, key: str, model: str, messages: list[dict]) -> str:
+    req = urllib.request.Request(
+        f"{base_url.rstrip('/')}/chat/completions",
+        data=json.dumps({"model": model, "messages": messages,
+                         "temperature": 0.7, "max_tokens": 1200}).encode(),
+        headers={"Content-Type": "application/json", "Authorization": f"Bearer {key}"},
+        method="POST",
+    )
+    with urllib.request.urlopen(req, timeout=180) as resp:
+        body = json.loads(resp.read().decode())
+    usage = body.get("usage") or {}
+    return body["choices"][0]["message"]["content"], usage
+
+
+def _parse_actions(reply: str) -> list[dict]:
+    """Extract the first JSON array from the reply; malformed output means no actions."""
+    start = reply.find("[")
+    if start < 0:
+        return []
+    depth = 0
+    for i, ch in enumerate(reply[start:], start):
+        if ch == "[":
+            depth += 1
+        elif ch == "]":
+            depth -= 1
+            if depth == 0:
+                try:
+                    parsed = json.loads(reply[start:i + 1])
+                except json.JSONDecodeError:
+                    return []
+                return [a for a in parsed if isinstance(a, dict)][:MAX_ACTIONS_PER_PHASE]
+    return []
+
+
+class LLMHarness(MinimalHarness):
+    """`minimal`'s surfaces and trace format, driven by a live model."""
+
+    name = "llm"
+
+    def __init__(self, model: str | None = None, base_url: str | None = None,
+                 key: str | None = None) -> None:
+        super().__init__(policy=None)  # the policy hook is unused; phases call the model
+        self.base_url = base_url or os.environ.get(
+            "PROTEUS_LLM_BASE_URL", "https://api.deepseek.com")
+        self.model = model or os.environ.get("PROTEUS_LLM_MODEL", "deepseek-chat")
+        self.key = key or (os.environ.get("PROTEUS_LLM_KEY")
+                           or os.environ.get("DEEPSEEK_KEY")
+                           or os.environ.get("OPENAI_API_KEY", ""))
+
+    def run_episode(self, spec: EpisodeSpec) -> EpisodeResult:
+        if not self.key:
+            return EpisodeResult(episode=spec.episode, ok=False, turns=0,
+                                 error="no API key: set DEEPSEEK_KEY / PROTEUS_LLM_KEY")
+        model = spec.model or self.model
+        harness = spec.root / "harness"
+        trace_path = spec.root / "traces" / f"ep{spec.episode:03d}.jsonl"
+        trace_path.parent.mkdir(parents=True, exist_ok=True)
+        turn = 0
+        writes = {"notes": 0, "tools": 0}
+        tokens_in = tokens_out = 0
+        error = ""
+        capped = False
+        min_pp = int(getattr(spec, "min_turns_per_phase", 0) or 0)
+        with trace_path.open("w", encoding="utf-8") as sink:
+            for idx, phase in enumerate(PHASES):
+                # `max_turns` bounds the whole episode, model calls included; the phase
+                # stop line additionally reserves min_turns_per_phase for later phases,
+                # and reaching it ends the phase, not the episode
+                if spec.max_turns and turn >= spec.max_turns:
+                    capped = True
+                if capped:
+                    break
+                stop_at = (spec.max_turns - min_pp * (len(PHASES) - idx - 1)
+                           if spec.max_turns else 0)
+                prompt = spec.phase_prompts.get(phase, "")
+                user = (f"Episode {spec.episode}, phase: {phase}.\n\n"
+                        f"{prompt}\n\nCurrent harness state:\n{_render_state(harness)}")
+                try:
+                    reply, usage = _chat(self.base_url, self.key, model,
+                                         [{"role": "system", "content": SYSTEM},
+                                          {"role": "user", "content": user}])
+                except Exception as exc:  # noqa: BLE001 - recorded, episode marked failed
+                    error = f"{type(exc).__name__}: {exc}"
+                    break
+                tokens_in += int(usage.get("prompt_tokens") or 0)
+                tokens_out += int(usage.get("completion_tokens") or 0)
+                turn += 1
+                sink.write(json.dumps({"turn": turn, "phase": phase, "tool": None,
+                                       "surface": None, "text": reply[:500]}) + "\n")
+                for act in _parse_actions(reply):
+                    if spec.max_turns and turn >= stop_at:
+                        capped = turn >= spec.max_turns
+                        break
+                    tool = act.get("tool", "")
+                    name = "".join(c for c in str(act.get("name", "unnamed"))
+                                   if c.isalnum() or c in "-_")[:60] or "unnamed"
+                    text = str(act.get("text", ""))
+                    turn += 1
+                    if tool == "write_note":
+                        (harness / "notes" / f"{name}.md").write_text(text, encoding="utf-8")
+                        writes["notes"] += 1
+                        surface = "notes"
+                    elif tool == "write_tool":
+                        (harness / "tools" / f"{name}.py").write_text(text, encoding="utf-8")
+                        writes["tools"] += 1
+                        surface = "tools"
+                    else:
+                        continue
+                    sink.write(json.dumps({"turn": turn, "phase": phase, "tool": tool,
+                                           "surface": surface, "text": name}) + "\n")
+        return EpisodeResult(episode=spec.episode, ok=not error, turns=turn, error=error,
+                             counters={"writes": writes, "turn_capped": capped,
+                                       "tokens_in": tokens_in, "tokens_out": tokens_out})

proteus/adapters/minimal.py

@@ -0,0 +1,141 @@
+"""A minimal, dependency-free reference harness — proof that Proteus is harness-agnostic.
+
+This harness is deliberately tiny: two surfaces (`notes`, `tools`), a four-phase loop, and
+a pluggable policy. It exists to (a) let the whole framework run end-to-end with no API key
+and no Docker, so tests and demos work offline, and (b) demonstrate that a harness with
+*its own* surface set plugs in with no change to the framework or the measurement layer.
+
+The default `MockPolicy` is deterministic and biases its writes toward whichever surface the
+installed disposition names, so even offline a `review_notes` run diverges from `neutral` in
+a way the distance instrument can measure. `proteus.adapters.llm.LLMHarness` is the same
+harness driven by a live model instead of the mock policy.
+"""
+
+from __future__ import annotations
+
+import json
+import random
+from pathlib import Path
+from typing import Callable, Optional, Sequence, Tuple
+
+from proteus.core.adapter import ActionEvent, EpisodeResult, EpisodeSpec, Surface
+from proteus.core.disposition import Disposition
+from proteus.core.episode import PHASES
+
+# A policy maps (phase, prompt, episode, rng) -> list of (tool, surface, text) actions.
+Action = Tuple[str, Optional[str], str]
+Policy = Callable[[str, str, int, random.Random], list[Action]]
+
+
+def mock_policy(phase: str, prompt: str, episode: int, rng: random.Random) -> list[Action]:
+    """Deterministic offline policy. Writes to the surface the prompt leans on.
+
+    It reads the disposition's leaning straight from the phase prompt text (the prompt is
+    all the real agent would see too), so a notes-disposition produces notes and a
+    tools-disposition produces tools — a measurable, reproducible effect without any model.
+    """
+    low = prompt.lower()
+    leans_notes = "notes" in low
+    leans_tools = "tools" in low
+    acts: list[Action] = []
+    if phase == "observe":
+        acts.append(("read_state", None, "surveyed the harness"))
+    elif phase == "act":
+        # baseline: occasionally write a note; disposition tilts the surface + rate.
+        if leans_tools or (not leans_notes and rng.random() < 0.3):
+            acts.append(("write_tool", "tools", f"tool_e{episode}"))
+        if leans_notes or rng.random() < 0.5:
+            acts.append(("write_note", "notes", f"note_e{episode}_{rng.randint(0,3)}"))
+    elif phase == "reflect" and leans_notes and rng.random() < 0.6:
+        acts.append(("write_note", "notes", f"reflection_e{episode}"))
+    return acts
+
+
+class MinimalHarness:
+    """A `HarnessAdapter` for the minimal reference harness."""
+
+    name = "minimal"
+    disposition_in_files = False   # the perturbation reaches this harness via prompts
+
+    def __init__(self, policy: Policy = mock_policy) -> None:
+        self.policy = policy
+
+    def surfaces(self) -> Sequence[Surface]:
+        return [
+            Surface("notes", "notes", unit="file", write_tools=frozenset({"write_note"})),
+            Surface("tools", "tools", unit="file", write_tools=frozenset({"write_tool"}),
+                    is_code=True),
+        ]
+
+    def required_edit_tools(self) -> frozenset[str]:
+        return frozenset({"write_note", "write_tool"})
+
+    def seed(self, harness_root: Path, rng_seed: int = 0) -> None:
+        for sub in ("notes", "tools"):
+            (harness_root / sub).mkdir(parents=True, exist_ok=True)
+        (harness_root / "STATE.md").write_text("# minimal harness\nepisode 0\n")
+
+    def install_disposition(self, harness_root: Path, disposition: Disposition) -> None:
+        (harness_root / ".disposition.json").write_text(json.dumps({
+            "label": disposition.label,
+            "config": dict(disposition.config),
+            "empty": disposition.is_empty,
+        }))
+
+    def run_episode(self, spec: EpisodeSpec) -> EpisodeResult:
+        harness = spec.root / "harness"
+        for sub in ("notes", "tools"):
+            # git snapshots do not track empty directories, so a restore after a rejected
+            # episode may drop them; an episode must tolerate waking up without them
+            (harness / sub).mkdir(exist_ok=True)
+        rng = random.Random(f"{spec.seed}:{spec.root.name}:{spec.episode}")
+        trace_path = spec.root / "traces" / f"ep{spec.episode:03d}.jsonl"
+        trace_path.parent.mkdir(parents=True, exist_ok=True)
+        turn = 0
+        writes = {"notes": 0, "tools": 0}
+        capped = False
+        min_pp = int(getattr(spec, "min_turns_per_phase", 0) or 0)
+        with trace_path.open("w", encoding="utf-8") as sink:
+            for idx, phase in enumerate(PHASES):
+                if capped:
+                    break
+                # the phase's stop line reserves min_turns_per_phase for every later
+                # phase; reaching it ends THIS phase, only a spent budget ends the episode
+                stop_at = (spec.max_turns - min_pp * (len(PHASES) - idx - 1)
+                           if spec.max_turns else 0)
+                prompt = spec.phase_prompts.get(phase, "")
+                for tool, surface, text in self.policy(phase, prompt, spec.episode, rng):
+                    if spec.max_turns and turn >= stop_at:
+                        capped = turn >= spec.max_turns
+                        break
+                    turn += 1
+                    if tool == "write_note":
+                        (harness / "notes" / f"{text}.md").write_text(
+                            f"episode {spec.episode}: {text}\n")
+                        writes["notes"] += 1
+                    elif tool == "write_tool":
+                        (harness / "tools" / f"{text}.py").write_text(
+                            f"# episode {spec.episode}\ndef {text}():\n    return {spec.episode}\n")
+                        writes["tools"] += 1
+                    sink.write(json.dumps({
+                        "turn": turn, "phase": phase, "tool": tool,
+                        "surface": surface, "text": text,
+                    }) + "\n")
+        return EpisodeResult(episode=spec.episode, ok=True, turns=turn,
+                             counters={"writes": writes, "turn_capped": capped})
+
+    def read_trace(self, root: Path, episode: int) -> Sequence[ActionEvent]:
+        path = root / "traces" / f"ep{episode:03d}.jsonl"
+        out: list[ActionEvent] = []
+        if not path.exists():
+            return out
+        for line in path.read_text(encoding="utf-8").splitlines():
+            d = json.loads(line)
+            out.append(ActionEvent(turn=d["turn"], phase=d["phase"], tool=d.get("tool"),
+                                   surface=d.get("surface"), text=d.get("text", "")))
+        return out
+
+    def disposition_fingerprint(self, harness_root: Path) -> str:
+        import hashlib
+        p = harness_root / ".disposition.json"
+        return hashlib.sha256(p.read_bytes()).hexdigest() if p.exists() else ""

proteus/adapters/pi.py

@@ -0,0 +1,317 @@
+"""Pi adapter — Mario Zechner's minimal coding harness (pi-mono) in a prepared container.
+
+Pi (github.com/badlogic/pi-mono, npm `@earendil-works/pi-coding-agent`) is the minimal
+end of the harness spectrum: four built-in tools (read/write/edit/bash), native
+`AGENTS.md` context loading, native skills. That makes it the cleanest demonstration that
+the adapter contract covers real third-party harnesses of any size — the whole adapter is
+symmetric with `dsh.py` and shares its disposition carrier.
+
+Per phase, one non-interactive pi session (`-p`) runs in the source-mode image from
+`environments/pi-src/`, with the workspace at `/workspace`, session/build state at
+`/state`, and an optional benchmark workspace at `/workspace/task`. Each run evolves the
+real Pi TypeScript source under `harness/src/`; the image exact-syncs and rebuilds it before
+boot. The trace is parsed from pi's session JSONL (v3: `message` events whose content blocks
+carry `toolCall` entries). Skills are loaded explicitly with `--skill /workspace/skills`,
+so the skills surface is version-robust rather than relying on discovery conventions.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import subprocess
+from pathlib import Path
+from typing import Optional, Sequence
+
+from proteus.adapters import instructions
+from proteus.core.adapter import ActionEvent, EpisodeResult, EpisodeSpec, Surface
+from proteus.core.disposition import Disposition
+from proteus.core.episode import PHASES
+
+IMAGE = os.environ.get("PROTEUS_PI_IMAGE", "proteus-env-pi-src:0.84.2")
+PHASE_TIMEOUT_S = 600
+#: The editable self is pi's real TypeScript source (~1,100 .ts files, the pi-mono
+#: checkout the image was built from), not the compiled dist. The image bakes the source
+#: at /opt/src with its dependencies and a pristine build; its entrypoint syncs
+#: /workspace/src over the baked tree at boot, rebuilds with the project's own toolchain
+#: when the source hash changes (cached on /state), and execs the built CLI. See
+#: environments/pi-src/.
+SOURCE_TAR = "/opt/pi-source.tar"
+
+SEED_INSTRUCTIONS = """\
+# Agent instructions
+
+You maintain and improve your own harness — the files in this workspace, which persist
+across sessions. Your surfaces:
+
+- `AGENTS.md` — these instructions (you may refine them)
+- `notes/` — markdown knowledge you want future sessions to have
+- `tools/` — small python utilities you may want later
+- `skills/` — pi skill files (loaded automatically next session)
+- `src/` — your own program: the real TypeScript source of the agent that runs you.
+  Edit it and the next session is rebuilt from your edit and runs it. If your edit
+  does not compile, the run ends.
+
+Each session is one phase of an episode; only these files carry over.
+"""
+
+
+class PiHarness:
+    """`HarnessAdapter` for pi-coding-agent's non-interactive mode, containerized."""
+
+    name = "pi"
+    disposition_in_files = True   # carried by AGENTS.md; keep it out of the phase prompts
+
+    SURFACES = (
+        Surface("instructions", "AGENTS.md", unit="file", free_named=False),
+        Surface("skills", "skills", unit="file", write_tools=frozenset({"write", "edit"})),
+        Surface("notes", "notes", unit="file", write_tools=frozenset({"write", "edit"})),
+        Surface("tools", "tools", unit="file", write_tools=frozenset({"write", "edit"}),
+                is_code=True),
+        # the harness's own program, shadow-mounted over the install path at boot — see
+        # DshHarness: same arrangement, same reasons
+        Surface("loop", "src", unit="file", is_code=True, free_named=False,
+                write_tools=frozenset({"write", "edit"})),
+    )
+
+    def __init__(self, image: str = IMAGE, network: str = "host",
+                 provider: str = "deepseek", model: str = "deepseek-v4-flash",
+                 key: str | None = None, sandbox=None,
+                 phase_timeout_s: int = PHASE_TIMEOUT_S) -> None:
+        self.image = image
+        self.network = network
+        self.provider = provider
+        self.model = model
+        self.phase_timeout_s = phase_timeout_s
+        # per-instance key injection first (multi-tenant runs must not share env)
+        self.key = key or os.environ.get("DEEPSEEK_API_KEY") or os.environ.get("DEEPSEEK_KEY", "")
+        from proteus.sandbox import DockerSandbox, SandboxConfig
+        # containers write into bind mounts; on Linux a root-in-container write leaves
+        # root-owned files the host user can neither snapshot-clean nor edit, so the
+        # container runs as the host user (the images chmod their /opt/src for this)
+        host_user = f"{os.getuid()}:{os.getgid()}" if hasattr(os, "getuid") else ""
+        # a caller may pass its own environment (see DshHarness.__init__)
+        self.sandbox = sandbox or DockerSandbox(SandboxConfig(
+            network=network, image=image, env_passthrough=("DEEPSEEK_API_KEY",),
+            user=host_user,
+        ))
+
+    def surfaces(self) -> Sequence[Surface]:
+        return self.SURFACES
+
+    def required_edit_tools(self) -> frozenset[str]:
+        return frozenset({"write", "edit"})
+
+    def seed(self, harness_root: Path, rng_seed: int = 0) -> None:
+        harness_root.mkdir(parents=True, exist_ok=True)
+        (harness_root / "AGENTS.md").write_text(SEED_INSTRUCTIONS, encoding="utf-8")
+        for sub in ("notes", "tools", "skills"):
+            (harness_root / sub).mkdir(exist_ok=True)
+        self._extract_self_code(harness_root / "src")
+
+    def _extract_self_code(self, dest: Path) -> None:
+        """Unpack the source the image was built from into `dest` (episode-0 state).
+
+        The image bakes a source-only tar at build time precisely so this is cheap and
+        exact: what the seed gets is byte-for-byte the source of the build it boots."""
+        if dest.exists() and any(dest.iterdir()):
+            return                        # resumed root: the seed owns its source already
+        dest.mkdir(parents=True, exist_ok=True)
+        user = (["--user", f"{os.getuid()}:{os.getgid()}"]
+                if hasattr(os, "getuid") else [])
+        proc = subprocess.run(
+            ["docker", "run", "--rm", "--network", "none", *user,
+             "-v", f"{dest}:/proteus-out", "--entrypoint", "sh", self.image,
+             "-c", f"tar -xf {SOURCE_TAR} -C /proteus-out --strip-components=1"],
+            capture_output=True, text=True, errors="replace", check=False)
+        if proc.returncode != 0:
+            raise RuntimeError(
+                f"could not extract pi source from {self.image}: {proc.stderr[-300:]}")
+
+    @staticmethod
+    def _task_mount(run_root: Path) -> tuple:
+        """Bind the run's task workspace (a snapshot-external sibling of the harness)
+        into the agent's view, when the run is goal-conditioned."""
+        task = run_root / "task"
+        return ((str(task), "/workspace/task"),) if task.is_dir() else ()
+
+    def check_boot(self, harness_root: Path) -> str:
+        """Viability gate: sync + rebuild + `--version` through the image's boot wrapper.
+
+        Because the wrapper rebuilds from /workspace/src, a type error the agent wrote
+        into its own source surfaces here as a legible build failure (exit 97 with the
+        build log tail), before any API spend."""
+        harness = Path(harness_root)
+        state = harness.parent / ".pi-state"
+        state.mkdir(exist_ok=True)
+        proc = self.sandbox.run(
+            harness.parent, ["--version"], env={}, timeout_s=300,
+            mounts=((str(harness), "/workspace"), (str(state), "/state")))
+        if proc.returncode != 0:
+            return (f"self-edited source does not boot (exit {proc.returncode}): "
+                    f"{(proc.stderr or proc.stdout)[-400:]}")
+        return ""
+
+    def install_disposition(self, harness_root: Path, disposition: Disposition) -> None:
+        instructions.install_block(harness_root / "AGENTS.md", disposition)
+
+    def disposition_fingerprint(self, harness_root: Path) -> str:
+        return instructions.block_fingerprint(Path(harness_root) / "AGENTS.md")
+
+    # ------------------------------------------------------------------ episodes
+
+    @staticmethod
+    def _sessions(state: Path) -> set[Path]:
+        return set(state.glob("*.jsonl"))
+
+    def run_episode(self, spec: EpisodeSpec) -> EpisodeResult:
+        if not self.key:
+            return EpisodeResult(episode=spec.episode, ok=False, turns=0,
+                                 error="no DeepSeek key: set DEEPSEEK_API_KEY")
+        run_root = Path(spec.root)
+        harness = run_root / "harness"
+        state = run_root / ".pi-state"
+        state.mkdir(exist_ok=True)
+        (run_root / "traces").mkdir(exist_ok=True)
+        mapping: dict[str, str] = {}
+        error = ""
+        capped = False
+        budget = int(spec.max_turns or 0)
+        episode_files: set = set()
+        if (harness / "src").is_dir():
+            error = self.check_boot(harness)
+        min_pp = int(getattr(spec, "min_turns_per_phase", 0) or 0)
+        for idx, phase in enumerate(PHASES if not error else ()):
+            # the budget is enforced twice, both harness-agnostically: exactly, between
+            # phases (no new phase once it is spent) and approximately, mid-phase (the
+            # session log is polled and the container stopped at the phase's stop line).
+            # The stop line reserves min_turns_per_phase for every later phase, so a
+            # reservation stop moves to the next phase; only a spent budget ends the
+            # episode.
+            if budget and self._live_calls(state, episode_files, set()) >= budget:
+                capped = True
+                break
+            stop_at = budget - min_pp * (len(PHASES) - idx - 1) if budget else 0
+            before = self._sessions(state)
+            fired = [False]
+
+            def stop_check(before=before, fired=fired, stop_at=stop_at):
+                if self._live_calls(state, episode_files,
+                                    self._sessions(state) - before) >= stop_at:
+                    fired[0] = True
+                    return True
+                return False
+
+            try:
+                proc = self.sandbox.run(
+                    run_root,
+                    ["--provider", self.provider, "--model", spec.model or self.model,
+                     "--session-dir", "/state", "--skill", "/workspace/skills",
+                     "-p", spec.phase_prompts.get(phase, phase)],
+                    env={"DEEPSEEK_API_KEY": self.key},
+                    timeout_s=self.phase_timeout_s,
+                    mounts=((str(harness), "/workspace"), (str(state), "/state"))
+                           + self._task_mount(run_root),
+                    stop_check=stop_check if budget else None,
+                )
+            except subprocess.TimeoutExpired:
+                error = f"phase {phase}: timeout after {self.phase_timeout_s}s"
+                break
+            new = self._sessions(state) - before
+            if new:
+                mapping[phase] = min(new).name
+                episode_files |= new
+            if proc.returncode != 0:
+                if fired[0]:
+                    # stopped at the phase's line: continue if it was only the reserve,
+                    # end the episode only when the whole budget is spent
+                    if budget and self._live_calls(state, episode_files, set()) >= budget:
+                        capped = True
+                        break
+                    continue
+                error = f"phase {phase}: exit {proc.returncode}: {proc.stderr[-400:]}"
+                break
+        (run_root / "traces" / f"ep{spec.episode:03d}.json").write_text(
+            json.dumps(mapping, indent=1))
+        trace = self.read_trace(run_root, spec.episode)
+        return EpisodeResult(
+            episode=spec.episode, ok=not error,
+            turns=sum(1 for e in trace if e.tool), error=error,
+            counters={"phases": len(mapping), "turn_capped": capped},
+        )
+
+    def _live_calls(self, state: Path, episode_files: set, extra: set) -> int:
+        """Tool calls made so far this episode, read live from the session logs.
+
+        pi's session JSONL is plain text and appended per event, so a mid-phase read
+        sees every call already made. Counted by marker substring — over-counting stops
+        early, which is the conservative direction for a budget."""
+        n = 0
+        for f in set(episode_files) | set(extra):
+            path = Path(f) if isinstance(f, Path) else state / f
+            try:
+                n += path.read_text(encoding="utf-8", errors="replace").count('"toolCall"')
+            except OSError:
+                continue
+        return n
+
+    # ------------------------------------------------------------------ measure path
+
+    def _surface_for_path(self, file_path: str) -> Optional[str]:
+        p = file_path.replace("/workspace/", "")
+        if p == "AGENTS.md":
+            return "instructions"
+        for s in ("skills", "notes", "tools"):
+            if p.startswith(f"{s}/"):
+                return s
+        if p.startswith("src/"):
+            return "loop"
+        return None
+
+    def read_trace(self, root: Path, episode: int) -> Sequence[ActionEvent]:
+        root = Path(root)
+        map_path = root / "traces" / f"ep{episode:03d}.json"
+        if not map_path.exists():
+            return []
+        mapping = json.loads(map_path.read_text())
+        state = root / ".pi-state"
+        events: list[ActionEvent] = []
+        turn = 0
+        for phase in PHASES:
+            name = mapping.get(phase)
+            if not name or not (state / name).exists():
+                continue
+            for line in (state / name).read_text(encoding="utf-8",
+                                                 errors="replace").splitlines():
+                try:
+                    e = json.loads(line)
+                except json.JSONDecodeError:
+                    continue
+                if e.get("type") != "message":
+                    continue
+                msg = e.get("message", {})
+                if msg.get("role") != "assistant":
+                    continue
+                turn += 1
+                for block in msg.get("content", []):
+                    btype = block.get("type", "")
+                    if btype in ("toolCall", "tool_call", "toolUse"):
+                        args = block.get("arguments") or block.get("input") or {}
+                        if isinstance(args, str):
+                            try:
+                                args = json.loads(args)
+                            except json.JSONDecodeError:
+                                args = {}
+                        path = str(args.get("file_path") or args.get("path") or "")
+                        events.append(ActionEvent(
+                            turn=turn, phase=phase,
+                            tool=block.get("name", ""),
+                            surface=self._surface_for_path(path),
+                            params={k: str(v)[:200] for k, v in args.items()}, text="",
+                        ))
+                    elif btype == "text" and block.get("text"):
+                        events.append(ActionEvent(
+                            turn=turn, phase=phase, tool=None, surface=None,
+                            params={}, text=block["text"][:500],
+                        ))
+        return events

proteus/bench/__init__.py

@@ -0,0 +1,23 @@
+"""Benchmarks as goals: task packs whose graders are Proteus evaluators."""
+from proteus.bench.task import (
+    TASK_SUBDIR,
+    BenchTask,
+    as_evaluator,
+    as_goal,
+    seed_task,
+    task_root,
+    workspace_diff,
+)
+
+__all__ = [
+    "TASK_SUBDIR",
+    "BenchTask",
+    "as_evaluator",
+    "as_goal",
+    "seed_task",
+    "task_root",
+    "workspace_diff",
+]
+
+from proteus.bench.polyglot import list_tasks, polyglot_task  # noqa: E402,F401
+__all__ += ["list_tasks", "polyglot_task"]