benchkit-cli 0.1.0__py3-none-any.whl

benchkit/sandbox.py

@@ -0,0 +1,1167 @@
+"""Persistent Docker environments for unmodified coding-agent harnesses.
+
+Builds the images described in :mod:`benchkit.dockerfiles`, runs Pi tasks and
+mc-arena scripts in them, and removes everything a run created.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import hashlib
+import json
+import os
+import re
+import shutil
+import subprocess
+import tempfile
+import threading
+import uuid
+from dataclasses import dataclass, field
+from pathlib import Path, PurePosixPath
+from urllib.parse import urlsplit, urlunsplit
+
+from benchkit.client import InferenceClient, _openai_base
+from benchkit.dockerfiles import (
+    AIDER_PI_DOCKERFILE,
+    GIT_SURGERY_PI_DOCKERFILE,
+    PI_DOCKERFILE,
+    PatchEvalRuntimeRecipe,
+    mc_arena_dockerfile,
+    patcheval_dockerfile,
+)
+
+# The Aider suite reads its pinned commit from here.
+from benchkit.dockerfiles import AIDER_POLYGLOT_COMMIT as AIDER_POLYGLOT_COMMIT
+
+_PI_IMAGE_SESSION = uuid.uuid4().hex[:12]
+PI_IMAGE = f"benchkit-pi:{_PI_IMAGE_SESSION}"
+AIDER_PI_IMAGE = f"benchkit-pi-aider-polyglot:{_PI_IMAGE_SESSION}"
+GIT_SURGERY_PI_IMAGE = f"benchkit-pi-git-surgery:{_PI_IMAGE_SESSION}"
+PI_PACKAGE = "@earendil-works/pi-coding-agent@0.84.2"
+PI_VERSION = "0.84.2"
+_PROXY_SOURCE = Path(__file__).with_name("_pi_proxy.py")
+_ANSWER_KEY_GUARD_SOURCE = Path(__file__).with_name("answer_key_guard.ts")
+_PI_PACKAGE_ROOT = Path(__file__).with_name("pi_package")
+_PATCHEVAL_IMAGE_SESSION = uuid.uuid4().hex[:12]
+_MC_ARENA_IMAGE_SESSION = uuid.uuid4().hex[:12]
+_BUILDKIT_DRIVER_IMAGE = "moby/buildkit:buildx-stable-1"
+
+# One label scopes every image, container, network, volume, and build-cache
+# entry that this process creates, so cleanup can remove exactly its own
+# resources instead of pruning Docker globally.
+RUN_ID = uuid.uuid4().hex[:16]
+RUN_LABEL = f"benchkit.run={RUN_ID}"
+MANAGED_LABEL = "benchkit.managed=true"
+_BUILDER = f"benchkit-build-{RUN_ID}"
+_BUILDKIT_CONTAINER = f"buildx_buildkit_{_BUILDER}0"
+_BUILDKIT_VOLUME = f"buildx_buildkit_{_BUILDER}0_state"
+_UV_CACHE_ID = f"benchkit-uv-{RUN_ID}"
+_BUILD_LOCK = threading.Lock()
+_BUILD_STATE: dict[str, object] = {
+    "builder": False,
+    "driver_was_present": True,
+    "pulled": set(),
+}
+
+
+def resource_labels(owner_id: str | None = None) -> list[str]:
+    """Return the label flags every managed Docker resource must carry."""
+    labels = ["--label", MANAGED_LABEL, "--label", RUN_LABEL]
+    if owner_id is not None:
+        labels.extend(["--label", f"benchkit.owner={owner_id}"])
+    return labels
+
+
+class SandboxError(RuntimeError):
+    """The Docker-backed agent environment could not be prepared or used."""
+
+
+def _docker_binary() -> str:
+    requested = os.environ.get("BENCHKIT_DOCKER", "docker")
+    resolved = shutil.which(requested)
+    if resolved is None:
+        raise SandboxError(
+            "Pi harness requires Docker; install Docker and ensure `docker` is on PATH"
+        )
+    return resolved
+
+
+def _tail(text: str, limit: int = 4000) -> str:
+    text = text.strip()
+    return text[-limit:] if len(text) > limit else text
+
+
+def _run(
+    args: list[str],
+    *,
+    input_text: str | None = None,
+    timeout: float | None = None,
+    check: bool = True,
+) -> subprocess.CompletedProcess[str]:
+    try:
+        completed = subprocess.run(
+            args,
+            input=input_text,
+            text=True,
+            errors="replace",
+            capture_output=True,
+            timeout=timeout,
+        )
+    except subprocess.TimeoutExpired as exc:
+        raise SandboxError(f"Docker command timed out: {args[1]}") from exc
+    except OSError as exc:
+        raise SandboxError(f"Could not run Docker: {exc}") from exc
+    if check and completed.returncode:
+        detail = _tail(completed.stderr or completed.stdout) or "unknown Docker error"
+        raise SandboxError(f"Docker {args[1]} failed: {detail}")
+    return completed
+
+
+def _docker_upstream(url: str) -> str:
+    """Make a host-local inference URL reachable from the proxy container."""
+    parsed = urlsplit(_openai_base(url))
+    host = parsed.hostname or ""
+    if host not in {"localhost", "127.0.0.1", "::1"}:
+        return urlunsplit(parsed)
+    replacement = "host.docker.internal"
+    if parsed.port:
+        replacement += f":{parsed.port}"
+    return urlunsplit((parsed.scheme, replacement, parsed.path, parsed.query, ""))
+
+
+def cleanup_owned_resources(docker: str, owner_id: str) -> None:
+    """Remove every container and network belonging to one Pi runner."""
+    label = f"benchkit.owner={owner_id}"
+    errors: list[str] = []
+
+    def listed(command: list[str], kind: str) -> list[str]:
+        try:
+            result = _run(command, timeout=30, check=False)
+        except Exception as exc:
+            errors.append(f"could not inspect owned {kind}: {exc}")
+            return []
+        if result.returncode:
+            detail = _tail(result.stderr or result.stdout) or "unknown Docker error"
+            errors.append(f"could not inspect owned {kind}: {detail}")
+            return []
+        return result.stdout.split()
+
+    container_query = [
+        docker,
+        "ps",
+        "--all",
+        "--quiet",
+        "--filter",
+        f"label={label}",
+    ]
+    network_query = [
+        docker,
+        "network",
+        "ls",
+        "--quiet",
+        "--filter",
+        f"label={label}",
+    ]
+    containers = listed(container_query, "containers")
+    if containers:
+        with contextlib.suppress(Exception):
+            _run([docker, "rm", "--force", *containers], timeout=30, check=False)
+    networks = listed(network_query, "networks")
+    if networks:
+        with contextlib.suppress(Exception):
+            _run([docker, "network", "rm", *networks], timeout=30, check=False)
+
+    remaining_containers = listed(container_query, "containers after cleanup")
+    remaining_networks = listed(network_query, "networks after cleanup")
+    if remaining_containers:
+        errors.append("owned containers remain: " + ", ".join(remaining_containers))
+    if remaining_networks:
+        errors.append("owned networks remain: " + ", ".join(remaining_networks))
+    if errors:
+        raise SandboxError("Pi Docker cleanup failed: " + "; ".join(errors))
+
+
+def _verify_absent(
+    command: list[str], description: str, errors: list[str], *, timeout: int = 30
+) -> None:
+    """Record an error unless an exact Docker resource is confirmed absent."""
+    try:
+        result = _run(command, timeout=timeout, check=False)
+    except Exception as exc:
+        errors.append(f"could not verify {description}: {exc}")
+        return
+    if result.returncode == 0:
+        errors.append(f"{description} remains")
+
+
+@dataclass
+class LatestPiImage:
+    """Build a reproducibly pinned Pi image; the engine builds one per task."""
+
+    docker: str = field(default_factory=_docker_binary)
+    image: str = PI_IMAGE
+    dockerfile: str = PI_DOCKERFILE
+    transient: bool = True
+    pids_limit: int = 256
+    build_assets: Path | None = None
+    build_files: tuple[tuple[Path, str], ...] = ()
+    always_cleanup_image: bool = True
+    resource_scope: str = "pi"
+    answer_key_guard: bool = False
+    version: str = ""
+    _ready: bool = field(default=False, init=False, repr=False)
+    _lock: threading.Lock = field(
+        default_factory=threading.Lock, init=False, repr=False
+    )
+
+    def prepare(self) -> str:
+        if self._ready:
+            return self.version
+        with self._lock:
+            if self._ready:
+                return self.version
+            _run(
+                [self.docker, "version", "--format", "{{.Server.Version}}"],
+                timeout=10,
+            )
+            with tempfile.TemporaryDirectory(prefix="benchkit-pi-build-") as directory:
+                context = Path(directory)
+                (context / "Dockerfile").write_text(self.dockerfile, encoding="utf-8")
+                (context / "inference_proxy.py").write_text(
+                    _PROXY_SOURCE.read_text(encoding="utf-8"),
+                    encoding="utf-8",
+                )
+                (context / "answer_key_guard.ts").write_text(
+                    _ANSWER_KEY_GUARD_SOURCE.read_text(encoding="utf-8"),
+                    encoding="utf-8",
+                )
+                shutil.copytree(_PI_PACKAGE_ROOT, context / "pi-package")
+                if self.build_assets is not None:
+                    shutil.copytree(self.build_assets, context / "git-surgery")
+                allowed = {
+                    "Dockerfile",
+                    "inference_proxy.py",
+                    "answer_key_guard.ts",
+                    "pi-package",
+                    "git-surgery",
+                }
+                for source, destination in self.build_files:
+                    target = PurePosixPath(destination)
+                    if (
+                        target.is_absolute()
+                        or len(target.parts) != 1
+                        or target.name in allowed
+                        or source.is_symlink()
+                        or not source.is_file()
+                    ):
+                        raise SandboxError("invalid explicit Pi build-context file")
+                    shutil.copyfile(source, context / target.name)
+                    allowed.add(target.name)
+                dockerignore = ["*", *sorted(f"!{item}" for item in allowed)]
+                dockerignore.extend(("!pi-package/**", "!git-surgery/**"))
+                (context / ".dockerignore").write_text(
+                    "\n".join(dockerignore) + "\n", encoding="utf-8"
+                )
+                try:
+                    _build_with_run_builder(
+                        self.docker,
+                        self.image,
+                        self.dockerfile,
+                        context,
+                    )
+                    smoke = (
+                        f"benchkit-{self.resource_scope}-smoke-{uuid.uuid4().hex[:16]}"
+                    )
+                    try:
+                        result = _run(
+                            [
+                                self.docker,
+                                "run",
+                                "--name",
+                                smoke,
+                                "--rm",
+                                *resource_labels(),
+                                self.image,
+                                "pi",
+                                "--version",
+                            ],
+                            timeout=30,
+                        )
+                    finally:
+                        with contextlib.suppress(Exception):
+                            _run(
+                                [self.docker, "rm", "--force", smoke],
+                                timeout=30,
+                                check=False,
+                            )
+                    self.version = result.stdout.strip() or "latest"
+                    if self.version != PI_VERSION:
+                        raise SandboxError(
+                            f"Pi image reported {self.version!r}; "
+                            f"expected {PI_VERSION!r}"
+                        )
+                except BaseException:
+                    if self.transient:
+                        with contextlib.suppress(Exception):
+                            _run(
+                                [
+                                    self.docker,
+                                    "image",
+                                    "rm",
+                                    "--force",
+                                    self.image,
+                                ],
+                                timeout=60,
+                                check=False,
+                            )
+                    self.version = ""
+                    raise
+            self._ready = True
+            return self.version
+
+    def cleanup(self) -> None:
+        """Remove the transient Pi image after its benchmark run."""
+        if not self.transient or (not self._ready and not self.always_cleanup_image):
+            return
+        try:
+            _run(
+                [self.docker, "image", "rm", "--force", self.image],
+                timeout=60,
+                check=not self.always_cleanup_image,
+            )
+            if self.always_cleanup_image:
+                errors: list[str] = []
+                _verify_absent(
+                    [self.docker, "image", "inspect", self.image],
+                    f"transient image {self.image}",
+                    errors,
+                )
+                if errors:
+                    raise SandboxError("Pi Docker cleanup failed: " + "; ".join(errors))
+        finally:
+            self._ready = False
+            self.version = ""
+
+
+def _base_image_refs(dockerfile: str) -> tuple[str, ...]:
+    """Return the external images a Dockerfile pulls, ignoring its own stages."""
+    stages: set[str] = set()
+    refs: list[str] = []
+    for line in dockerfile.splitlines():
+        match = re.match(r"\s*FROM\s+(\S+)(?:\s+[Aa][Ss]\s+(\S+))?\s*$", line)
+        if match is None:
+            continue
+        reference, alias = match.group(1), match.group(2)
+        if reference not in stages and reference not in refs:
+            refs.append(reference)
+        if alias:
+            stages.add(alias)
+    return tuple(refs)
+
+
+def _run_builder(docker: str) -> str:
+    """Create this run's single private Buildx builder on first use.
+
+    Every build in the run shares one builder, so layers and cache mounts are
+    reused instead of rebuilt per task. The builder keeps its BuildKit state in
+    its own container and volume, which cleanup removes by exact name. That is
+    what keeps a run's build cache separable from the host's own cache without
+    ever running a global prune.
+    """
+    with _BUILD_LOCK:
+        if not _BUILD_STATE["builder"]:
+            _BUILD_STATE["driver_was_present"] = bool(
+                _run(
+                    [
+                        docker,
+                        "image",
+                        "inspect",
+                        "--format",
+                        "{{.Id}}",
+                        _BUILDKIT_DRIVER_IMAGE,
+                    ],
+                    timeout=30,
+                    check=False,
+                ).stdout.strip()
+            )
+            _run(
+                [
+                    docker,
+                    "buildx",
+                    "create",
+                    "--name",
+                    _BUILDER,
+                    "--driver",
+                    "docker-container",
+                    "--driver-opt",
+                    f"image={_BUILDKIT_DRIVER_IMAGE}",
+                ],
+                timeout=60,
+            )
+            _BUILD_STATE["builder"] = True
+    return _BUILDER
+
+
+def _build_with_run_builder(
+    docker: str,
+    image: str,
+    dockerfile: str,
+    context: Path,
+) -> None:
+    """Build one image on the run's shared builder, pulling bases once."""
+    builder = _run_builder(docker)
+    with _BUILD_LOCK:
+        pulled = _BUILD_STATE["pulled"]
+        assert isinstance(pulled, set)
+        fresh = [
+            reference
+            for reference in _base_image_refs(dockerfile)
+            if reference not in pulled
+        ]
+        pulled.update(fresh)
+    _run(
+        [docker, "image", "rm", "--force", image],
+        timeout=60,
+        check=False,
+    )
+    build = [docker, "buildx", "build", "--builder", builder]
+    if fresh:
+        # Recipe and harness tags are pinned, so one pull per run is enough.
+        build.append("--pull")
+    build.extend(
+        [
+            "--label",
+            MANAGED_LABEL,
+            "--label",
+            RUN_LABEL,
+            "--load",
+            "--tag",
+            image,
+            str(context),
+        ]
+    )
+    _run(build, timeout=1800)
+
+
+def _remove_labelled(docker: str, errors: list[str]) -> None:
+    """Remove every container, network, volume, and image of this run."""
+    queries = (
+        (
+            "containers",
+            [docker, "ps", "--all", "--quiet", "--filter", f"label={RUN_LABEL}"],
+            lambda ids: [docker, "rm", "--force", *ids],
+        ),
+        (
+            "networks",
+            [docker, "network", "ls", "--quiet", "--filter", f"label={RUN_LABEL}"],
+            lambda ids: [docker, "network", "rm", *ids],
+        ),
+        (
+            "volumes",
+            [docker, "volume", "ls", "--quiet", "--filter", f"label={RUN_LABEL}"],
+            lambda ids: [docker, "volume", "rm", "--force", *ids],
+        ),
+        (
+            "images",
+            [docker, "image", "ls", "--quiet", "--filter", f"label={RUN_LABEL}"],
+            lambda ids: [docker, "image", "rm", "--force", *ids],
+        ),
+    )
+    for kind, query, removal in queries:
+        try:
+            listed = _run(query, timeout=30, check=False)
+        except Exception as exc:
+            errors.append(f"could not inspect run {kind}: {exc}")
+            continue
+        if listed.returncode:
+            detail = _tail(listed.stderr or listed.stdout) or "unknown Docker error"
+            errors.append(f"could not inspect run {kind}: {detail}")
+            continue
+        identifiers = sorted(set(listed.stdout.split()))
+        if not identifiers:
+            continue
+        with contextlib.suppress(Exception):
+            _run(removal(identifiers), timeout=120, check=False)
+        remaining = _run(query, timeout=30, check=False)
+        if remaining.returncode == 0 and remaining.stdout.split():
+            errors.append(
+                f"run {kind} remain: "
+                + ", ".join(sorted(set(remaining.stdout.split())))
+            )
+
+
+def cleanup_run_resources(docker: str | None = None) -> None:
+    """Tear down every Docker resource this run created, and nothing else."""
+    global _MC_ARENA_READY
+    docker = docker or _docker_binary()
+    errors: list[str] = []
+    # The mc-arena image carries this run's label, so the cleanup below deletes
+    # it. Forget that it was built, or a later job in the same process hands
+    # `docker run` a tag that no longer exists and every task is a harness
+    # error.
+    with _MC_ARENA_LOCK:
+        _MC_ARENA_READY = False
+    with _BUILD_LOCK:
+        builder_created = bool(_BUILD_STATE["builder"])
+        driver_was_present = bool(_BUILD_STATE["driver_was_present"])
+        _BUILD_STATE["builder"] = False
+        pulled = _BUILD_STATE["pulled"]
+        assert isinstance(pulled, set)
+        pulled.clear()
+    if builder_created:
+        # Removing the builder drops its BuildKit state volume, and with it
+        # every layer and uv cache entry the run created.
+        cleanup_commands = (
+            ([docker, "buildx", "rm", "--force", _BUILDER], 60),
+            ([docker, "rm", "--force", _BUILDKIT_CONTAINER], 30),
+            ([docker, "volume", "rm", "--force", _BUILDKIT_VOLUME], 30),
+        )
+        for command, timeout in cleanup_commands:
+            with contextlib.suppress(Exception):
+                _run(command, timeout=timeout, check=False)
+        if not driver_was_present:
+            with contextlib.suppress(Exception):
+                _run(
+                    [docker, "image", "rm", "--force", _BUILDKIT_DRIVER_IMAGE],
+                    timeout=60,
+                    check=False,
+                )
+    _remove_labelled(docker, errors)
+    if builder_created:
+        _verify_absent(
+            [docker, "buildx", "inspect", _BUILDER],
+            f"Buildx builder {_BUILDER}",
+            errors,
+        )
+        _verify_absent(
+            [docker, "container", "inspect", _BUILDKIT_CONTAINER],
+            f"BuildKit container {_BUILDKIT_CONTAINER}",
+            errors,
+        )
+        _verify_absent(
+            [docker, "volume", "inspect", _BUILDKIT_VOLUME],
+            f"BuildKit volume {_BUILDKIT_VOLUME}",
+            errors,
+        )
+        if not driver_was_present:
+            _verify_absent(
+                [docker, "image", "inspect", _BUILDKIT_DRIVER_IMAGE],
+                f"BuildKit driver image {_BUILDKIT_DRIVER_IMAGE}",
+                errors,
+            )
+    if errors:
+        raise SandboxError("Pi Docker cleanup failed: " + "; ".join(errors))
+
+
+MC_ARENA_IMAGE = f"benchkit-mc-arena:{_MC_ARENA_IMAGE_SESSION}"
+# Enough to diagnose a traceback; a script cannot buy more by printing more.
+STDERR_LIMIT = 256 * 1024
+
+
+_MC_ARENA_READY = False
+_MC_ARENA_USERS = 0
+_MC_ARENA_LOCK = threading.Lock()
+
+
+def run_resources_created() -> bool:
+    """Whether this process built anything the run-level sweep must remove."""
+    with _BUILD_LOCK:
+        return bool(_BUILD_STATE["builder"])
+
+
+@contextlib.contextmanager
+def _mc_arena_image_lease():
+    """Hold the mc-arena image for one script; the last holder removes it.
+
+    Each task builds the image (a cache hit on the run's builder), runs its
+    script, and deletes it, so a run never keeps the image between tasks.
+    """
+    global _MC_ARENA_USERS, _MC_ARENA_READY
+    with _MC_ARENA_LOCK:
+        _MC_ARENA_USERS += 1
+    try:
+        yield mc_arena_image()
+    finally:
+        with _MC_ARENA_LOCK:
+            _MC_ARENA_USERS -= 1
+            release = (
+                _MC_ARENA_USERS == 0
+                and _MC_ARENA_READY
+                and not os.environ.get("BENCHKIT_MC_ARENA_IMAGE", "").strip()
+            )
+            if release:
+                _MC_ARENA_READY = False
+                with contextlib.suppress(Exception):
+                    _run(
+                        [_docker_binary(), "image", "rm", "--force", MC_ARENA_IMAGE],
+                        timeout=60,
+                        check=False,
+                    )
+
+
+def mc_arena_image() -> str:
+    """Build the mc-arena script runner once per process and return its tag.
+
+    ``BENCHKIT_MC_ARENA_IMAGE`` names an already-built image instead, for hosts
+    that cannot build one (no registry route, no build network). It has to
+    provide ``uv`` on PATH and a writable ``/work``.
+    """
+    prebuilt = os.environ.get("BENCHKIT_MC_ARENA_IMAGE", "").strip()
+    if prebuilt:
+        return prebuilt
+
+    global _MC_ARENA_READY
+    if _MC_ARENA_READY:
+        return MC_ARENA_IMAGE
+    docker = _docker_binary()
+    with _MC_ARENA_LOCK:
+        if _MC_ARENA_READY:
+            return MC_ARENA_IMAGE
+        _run([docker, "version", "--format", "{{.Server.Version}}"], timeout=10)
+        dockerfile = mc_arena_dockerfile()
+        with tempfile.TemporaryDirectory(prefix="benchkit-mc-arena-build-") as name:
+            context = Path(name)
+            (context / "Dockerfile").write_text(dockerfile, encoding="utf-8")
+            _build_with_run_builder(docker, MC_ARENA_IMAGE, dockerfile, context)
+        _MC_ARENA_READY = True
+    return MC_ARENA_IMAGE
+
+
+@dataclass
+class _CappedStream:
+    """Drain one pipe in a thread, keeping at most ``limit`` characters.
+
+    Truncating after the fact does not help against a script that prints
+    forever: the whole stream would already be sitting in this process's
+    memory, where the container's own memory limit does not reach. Everything
+    past the cap is still read and dropped, so the writer never blocks on a
+    full pipe while its deadline runs out.
+    """
+
+    stream: object
+    limit: int
+    chunks: list[str] = field(default_factory=list)
+    kept: int = 0
+    truncated: bool = False
+
+    def pump(self) -> None:
+        try:
+            while True:
+                chunk = self.stream.read(65536)
+                if not chunk:
+                    break
+                if self.kept >= self.limit:
+                    self.truncated = True
+                    continue
+                keep = chunk[: self.limit - self.kept]
+                self.chunks.append(keep)
+                self.kept += len(keep)
+                if len(keep) < len(chunk):
+                    self.truncated = True
+        except (OSError, ValueError):  # pragma: no cover - pipe torn down
+            pass
+        finally:
+            with contextlib.suppress(Exception):
+                self.stream.close()
+
+    @property
+    def text(self) -> str:
+        return "".join(self.chunks)
+
+
+def _run_capped(
+    args: list[str],
+    *,
+    timeout: float,
+    stdout_limit: int,
+    stderr_limit: int,
+) -> tuple[int, _CappedStream, _CappedStream]:
+    """Run a command, keeping only bounded prefixes of its two streams."""
+    try:
+        process = subprocess.Popen(
+            args,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True,
+            errors="replace",
+        )
+    except OSError as exc:
+        raise SandboxError(f"Could not run Docker: {exc}") from exc
+
+    out = _CappedStream(process.stdout, stdout_limit)
+    err = _CappedStream(process.stderr, stderr_limit)
+    pumps = [threading.Thread(target=stream.pump, daemon=True) for stream in (out, err)]
+    for pump in pumps:
+        pump.start()
+    try:
+        process.wait(timeout=timeout)
+    except subprocess.TimeoutExpired:
+        process.kill()
+        process.wait()
+    for pump in pumps:
+        pump.join(timeout=10)
+    return process.returncode, out, err
+
+
+@dataclass(frozen=True)
+class ScriptRun:
+    """One execution of a model-written script inside the sandbox."""
+
+    exit_code: int
+    stdout: str
+    stderr: str
+    timed_out: bool = False
+    # Set when the script printed more than the sandbox was willing to keep.
+    stdout_truncated: bool = False
+
+    @property
+    def ok(self) -> bool:
+        return self.exit_code == 0 and not self.timed_out
+
+
+def run_python_script(
+    code: str,
+    *,
+    timeout_s: float = 30.0,
+    stdout_limit: int = 8 * 1024 * 1024,
+) -> ScriptRun:
+    """Run one untrusted Python script in a throwaway, network-less container.
+
+    The container has no network, no capabilities and a read-only root
+    filesystem with small writable tmpfs mounts, plus hard memory, process and
+    time limits: a script that forks, allocates or spins is bounded by the
+    sandbox rather than by anything the script itself agrees to.
+    """
+    with _mc_arena_image_lease() as image:
+        return _run_python_script_in(
+            image, code, timeout_s=timeout_s, stdout_limit=stdout_limit
+        )
+
+
+def _run_python_script_in(
+    image: str,
+    code: str,
+    *,
+    timeout_s: float,
+    stdout_limit: int,
+) -> ScriptRun:
+    docker = _docker_binary()
+    container = f"benchkit-mc-arena-{uuid.uuid4().hex[:12]}"
+    memory = os.environ.get("BENCHKIT_MC_ARENA_MEMORY", "512m")
+    cpus = os.environ.get("BENCHKIT_MC_ARENA_CPUS", "1")
+    pids = os.environ.get("BENCHKIT_MC_ARENA_PIDS", "64")
+
+    try:
+        _run(
+            [
+                docker,
+                "run",
+                "--detach",
+                "--name",
+                container,
+                *resource_labels(),
+                "--network",
+                "none",
+                "--cap-drop",
+                "ALL",
+                "--security-opt",
+                "no-new-privileges",
+                "--read-only",
+                "--tmpfs",
+                "/tmp:size=64m",
+                "--tmpfs",
+                "/home/runner:size=64m,uid=1000,gid=1000",
+                "--tmpfs",
+                "/work:size=16m,uid=1000,gid=1000",
+                "--pids-limit",
+                pids,
+                "--memory",
+                memory,
+                "--cpus",
+                cpus,
+                image,
+                "sleep",
+                "infinity",
+            ],
+            timeout=120,
+        )
+        # `docker cp` refuses a read-only rootfs even when the destination is a
+        # tmpfs, so the script is written from inside the container instead.
+        _run(
+            [docker, "exec", "--interactive", container, "tee", "/work/build.py"],
+            input_text=code,
+            timeout=60,
+        )
+        # Read the streams under a hard cap rather than buffering whatever the
+        # script decides to print: the container's memory limit bounds the
+        # script, not this process's copy of its output.
+        code_out, out, err = _run_capped(
+            [
+                docker,
+                "exec",
+                "--workdir",
+                "/work",
+                container,
+                "timeout",
+                "--signal=KILL",
+                f"{timeout_s:g}s",
+                "uv",
+                "run",
+                "--no-project",
+                "--offline",
+                "build.py",
+            ],
+            timeout=timeout_s + 60,
+            stdout_limit=stdout_limit,
+            stderr_limit=STDERR_LIMIT,
+        )
+    finally:
+        with contextlib.suppress(Exception):
+            _run([docker, "rm", "--force", container], timeout=60, check=False)
+
+    return ScriptRun(
+        exit_code=code_out,
+        stdout=out.text,
+        stdout_truncated=out.truncated,
+        stderr=_tail(err.text, 8000),
+        # `timeout` reports 137 for the KILL it sends once the deadline passes.
+        timed_out=code_out == 137,
+    )
+
+
+def aider_pi_image() -> LatestPiImage:
+    """Return an image definition containing every Aider Polyglot toolchain."""
+    return LatestPiImage(
+        image=AIDER_PI_IMAGE,
+        dockerfile=AIDER_PI_DOCKERFILE,
+        transient=True,
+        # cpp/bank-account creates 1,000 simultaneous std::threads. Linux
+        # accounts threads against Docker's PID cgroup, so the generic Pi
+        # sandbox limit of 256 makes the official test suite impossible.
+        pids_limit=2048,
+        answer_key_guard=True,
+        resource_scope="aider-polyglot",
+    )
+
+
+def git_surgery_pi_image() -> LatestPiImage:
+    """Return the stock Pi image with pinned Git and Git Surgery assets."""
+    return LatestPiImage(
+        image=GIT_SURGERY_PI_IMAGE,
+        dockerfile=GIT_SURGERY_PI_DOCKERFILE,
+        transient=True,
+        build_assets=Path(__file__).with_name("git_surgery"),
+        resource_scope="git-surgery",
+    )
+
+
+def _regular_file_sha256(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as handle:
+        for block in iter(lambda: handle.read(1024 * 1024), b""):
+            digest.update(block)
+    return digest.hexdigest()
+
+
+def _tree_sha256(root: Path) -> str:
+    digest = hashlib.sha256()
+    for path in sorted(root.rglob("*")):
+        if path.is_symlink():
+            raise ValueError("Pi package build assets must not contain symlinks")
+        if not path.is_file():
+            continue
+        relative = path.relative_to(root).as_posix().encode()
+        digest.update(len(relative).to_bytes(8, "big"))
+        digest.update(relative)
+        digest.update(bytes.fromhex(_regular_file_sha256(path)))
+    return digest.hexdigest()
+
+
+def patcheval_pi_image(
+    source_archive: Path,
+    source_sha256: str,
+    recipe: PatchEvalRuntimeRecipe,
+) -> LatestPiImage:
+    """Build one task runtime locally without retaining buildkit state."""
+    if source_archive.is_symlink() or not source_archive.is_file():
+        raise ValueError("PatchEval source archive must be a regular file")
+    if not re.fullmatch(r"[0-9a-f]{64}", source_sha256):
+        raise ValueError("PatchEval source_sha256 must be a lowercase SHA-256")
+    if _regular_file_sha256(source_archive) != source_sha256:
+        raise ValueError("PatchEval source archive checksum mismatch")
+    # Hash a run-independent Dockerfile so the same task keeps one identity
+    # across runs even though the uv cache mount is run-scoped.
+    dockerfile = patcheval_dockerfile(recipe, _UV_CACHE_ID)
+    identity_dockerfile = patcheval_dockerfile(recipe, "benchkit-uv")
+    recipe_json = json.dumps(
+        {
+            "schema_version": recipe.schema_version,
+            "base_image": recipe.base_image,
+            "sync_command": list(recipe.sync_command),
+            "bootstrap_command": list(recipe.bootstrap_command),
+            "environment": list(recipe.environment),
+        },
+        sort_keys=True,
+        separators=(",", ":"),
+    )
+    identity = hashlib.sha256()
+    for value in (
+        identity_dockerfile,
+        source_sha256,
+        recipe_json,
+        _tree_sha256(_PI_PACKAGE_ROOT),
+        _regular_file_sha256(_PROXY_SOURCE),
+        _regular_file_sha256(_ANSWER_KEY_GUARD_SOURCE),
+    ):
+        identity.update(value.encode())
+        identity.update(b"\0")
+    image_id = identity.hexdigest()[:16]
+    return LatestPiImage(
+        image=f"benchkit-pi-patcheval:{_PATCHEVAL_IMAGE_SESSION}-{image_id}",
+        dockerfile=dockerfile,
+        transient=True,
+        build_files=((source_archive, "parent-source.tar"),),
+        always_cleanup_image=True,
+        resource_scope="patcheval",
+    )
+
+
+@dataclass
+class DockerTaskEnvironment:
+    """One isolated, persistent workspace shared by every native Pi tool call."""
+
+    client: InferenceClient
+    model: str
+    image: LatestPiImage
+    docker: str = field(default_factory=_docker_binary)
+    owner_id: str = field(default_factory=lambda: uuid.uuid4().hex)
+    workdir: str = "/workspace"
+    task_name: str = field(default_factory=lambda: f"benchkit-{uuid.uuid4().hex[:12]}")
+    network_name: str = field(init=False)
+    proxy_name: str = field(init=False)
+    container_name: str = field(init=False)
+    _started: bool = field(default=False, init=False, repr=False)
+
+    def __post_init__(self) -> None:
+        self.network_name = f"{self.task_name}-net"
+        self.proxy_name = f"{self.task_name}-proxy"
+        self.container_name = f"{self.task_name}-agent"
+
+    def __enter__(self) -> DockerTaskEnvironment:
+        self.start()
+        return self
+
+    def __exit__(self, *_args: object) -> None:
+        self.stop()
+
+    def start(self) -> None:
+        if self._started:
+            return
+        if not self.image.version:
+            raise SandboxError("Pi image must be prepared before starting a task")
+        try:
+            labels = resource_labels(self.owner_id)
+            _run(
+                [
+                    self.docker,
+                    "network",
+                    "create",
+                    "--internal",
+                    *labels,
+                    self.network_name,
+                ]
+            )
+            proxy_args = [
+                self.docker,
+                "run",
+                "--detach",
+                "--name",
+                self.proxy_name,
+                *labels,
+                "--network",
+                self.network_name,
+                "--network-alias",
+                "inference",
+                "--add-host",
+                "host.docker.internal:host-gateway",
+                "--cap-drop",
+                "ALL",
+                "--security-opt",
+                "no-new-privileges",
+                "--pids-limit",
+                "64",
+                "--memory",
+                "256m",
+                "--env",
+                f"BENCHKIT_UPSTREAM={_docker_upstream(self.client.host)}",
+                "--env",
+                f"BENCHKIT_MODEL={self.model}",
+                "--env",
+                f"BENCHKIT_UPSTREAM_TIMEOUT={self.client.timeout}",
+            ]
+            if self.client.api_key:
+                proxy_args.extend(
+                    ["--env", f"BENCHKIT_UPSTREAM_API_KEY={self.client.api_key}"]
+                )
+            proxy_args.extend(
+                [self.image.image, "python3", "/opt/benchkit/inference_proxy.py"]
+            )
+            _run(proxy_args)
+            _run([self.docker, "network", "connect", "bridge", self.proxy_name])
+
+            memory = os.environ.get("BENCHKIT_SANDBOX_MEMORY", "2g")
+            cpus = os.environ.get("BENCHKIT_SANDBOX_CPUS", "2")
+            pids = os.environ.get("BENCHKIT_SANDBOX_PIDS", str(self.image.pids_limit))
+            _run(
+                [
+                    self.docker,
+                    "run",
+                    "--detach",
+                    "--name",
+                    self.container_name,
+                    *labels,
+                    "--network",
+                    self.network_name,
+                    "--cap-drop",
+                    "ALL",
+                    "--security-opt",
+                    "no-new-privileges",
+                    "--pids-limit",
+                    pids,
+                    "--memory",
+                    memory,
+                    "--cpus",
+                    cpus,
+                    self.image.image,
+                ]
+            )
+            self._write_model_config()
+            self._started = True
+        except Exception:
+            self.stop()
+            raise
+
+    def _write_model_config(self) -> None:
+        context_length = None
+        discover = getattr(self.client, "context_length", None)
+        if callable(discover):
+            context_length = discover(self.model)
+        model: dict[str, object] = {"id": self.model, "name": self.model}
+        if isinstance(context_length, int) and context_length > 0:
+            model["contextWindow"] = context_length
+
+        provider: dict[str, object] = {
+            "baseUrl": "http://inference:8080/v1",
+            "api": "openai-completions",
+            "apiKey": "benchkit-sandbox",
+            "models": [model],
+        }
+        if self.client.provider == "ollama":
+            provider["compat"] = {
+                "supportsDeveloperRole": False,
+                "supportsReasoningEffort": False,
+            }
+        payload = json.dumps({"providers": {"benchkit": provider}})
+        _run(
+            [
+                self.docker,
+                "exec",
+                "--interactive",
+                self.container_name,
+                "tee",
+                "/home/node/.pi/agent/models.json",
+            ],
+            input_text=payload,
+        )
+
+    def start_pi(self) -> subprocess.Popen[str]:
+        if not self._started:
+            raise SandboxError("task environment is not running")
+        command = [
+            self.docker,
+            "exec",
+            "--interactive",
+            "--workdir",
+            self.workdir,
+            self.container_name,
+            "pi",
+            "--mode",
+            "rpc",
+            "--no-session",
+            "--provider",
+            "benchkit",
+            "--model",
+            self.model,
+        ]
+        if self.image.answer_key_guard:
+            command.extend(["--extension", "/opt/benchkit/answer_key_guard.ts"])
+        return subprocess.Popen(
+            command,
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True,
+            bufsize=1,
+        )
+
+    def pi_scaffold(self) -> dict:
+        """Read scaffold metadata captured from Pi's first inference request."""
+        result = _run(
+            [
+                self.docker,
+                "exec",
+                self.proxy_name,
+                "cat",
+                "/tmp/benchkit_pi_scaffold.json",
+            ],
+            timeout=10,
+            check=False,
+        )
+        if result.returncode:
+            return {}
+        try:
+            payload = json.loads(result.stdout)
+        except (json.JSONDecodeError, TypeError):
+            return {}
+        return payload if isinstance(payload, dict) else {}
+
+    def exec(
+        self,
+        command: list[str],
+        *,
+        workdir: str | None = None,
+        input_text: str | None = None,
+        timeout: float | None = None,
+        check: bool = True,
+    ) -> subprocess.CompletedProcess[str]:
+        return _run(
+            [
+                self.docker,
+                "exec",
+                *(["--workdir", workdir] if workdir is not None else []),
+                self.container_name,
+                *command,
+            ],
+            input_text=input_text,
+            timeout=timeout,
+            check=check,
+        )
+
+    def upload(self, source: Path, destination: str) -> None:
+        _run([self.docker, "cp", str(source), f"{self.container_name}:{destination}"])
+
+    def download(self, source: str, destination: Path) -> None:
+        _run([self.docker, "cp", f"{self.container_name}:{source}", str(destination)])
+
+    def stop(self) -> None:
+        try:
+            commands = [
+                [self.docker, "rm", "--force", self.container_name],
+                [self.docker, "rm", "--force", self.proxy_name],
+                [self.docker, "network", "rm", self.network_name],
+            ]
+            for command in commands:
+                with contextlib.suppress(Exception):
+                    _run(command, timeout=30, check=False)
+        finally:
+            self._started = False