commitarena 0.1.1__py3-none-any.whl

commitarena/runner.py

@@ -0,0 +1,298 @@
+"""Run one task with one model, score it with the original tests, and run whole suites."""
+
+from __future__ import annotations
+
+import json
+import os
+import platform
+import shutil
+import subprocess
+import tempfile
+import time
+from collections.abc import Callable
+from dataclasses import asdict, dataclass, field
+from datetime import UTC, datetime
+from pathlib import Path
+
+from . import __version__
+from .agent import AgentConfig, run_agent
+from .errors import BudgetExceeded, CommitArenaError
+from .llm import Budget, ChatClient, ModelSpec
+from .sandbox import Sandbox, SandboxConfig, build_image, ensure_docker, image_exists
+from .tasks import (
+    Task,
+    changed_files,
+    extract_commit,
+    init_workspace_git,
+    is_test_path,
+    load_tasks,
+    resolve_repo,
+    save_tasks,
+    unified_diff,
+    write_files,
+)
+from .testrun import run_tests
+
+CONTEXT_NAMES = ("AGENTS.md", "CLAUDE.md")
+
+
+@dataclass
+class ContextFile:
+    """A context file (AGENTS.md, CLAUDE.md, ...) injected into the attempt.
+
+    in_repo: write it at the repo root inside the sandbox.
+    in_prompt: append its content to the system prompt.
+    """
+
+    content: str
+    name: str = "AGENTS.md"
+    in_repo: bool = True
+    in_prompt: bool = True
+
+    @classmethod
+    def from_path(cls, path: str | os.PathLike, mode: str = "both", name: str | None = None) -> ContextFile:
+        if mode not in ("both", "repo", "prompt"):
+            raise CommitArenaError("--context-mode must be one of: both, repo, prompt")
+        p = Path(path)
+        return cls(p.read_text(), name or p.name, mode in ("both", "repo"), mode in ("both", "prompt"))
+
+
+@dataclass
+class RunResult:
+    task_id: str
+    model: str
+    passed: bool
+    fail_to_pass_passed: int = 0
+    fail_to_pass_total: int = 0
+    pass_to_pass_failed: int = 0
+    pass_to_pass_total: int = 0
+    cost_usd: float = 0.0
+    tokens_in: int = 0
+    tokens_out: int = 0
+    steps: int = 0
+    tool_calls: int = 0
+    wall_time_s: float = 0.0
+    stop_reason: str = ""
+    error: str = ""
+    patch: str = ""  # model's diff on non-test files (what is scored)
+    ignored_test_edits: list[str] = field(default_factory=list)
+    task_date: str = ""
+    context_file: str | None = None
+    started_at: str = ""
+
+    def to_dict(self) -> dict:
+        return asdict(self)
+
+    @classmethod
+    def from_dict(cls, d: dict) -> RunResult:
+        return cls(**{k: v for k, v in d.items() if k in cls.__dataclass_fields__})
+
+
+def ensure_task_image(task: Task) -> None:
+    if image_exists(task.image):
+        return
+    repo = resolve_repo(task.repo)
+    tmp = Path(tempfile.mkdtemp(prefix="commitarena-img-"))
+    try:
+        extract_commit(repo, task.commit, tmp / "snap")
+        build_image(task.image, task.env_spec, tmp / "snap", tmp)
+    finally:
+        shutil.rmtree(tmp, ignore_errors=True)
+
+
+def score_workspace(task: Task, pristine: Path, agent_tree: Path, sandbox: SandboxConfig, test_timeout: int):
+    """Apply only the agent's non-test changes onto the pristine workspace and run the suite."""
+    changes = changed_files(pristine, agent_tree)
+    code = {p: c for p, c in changes.items() if not is_test_path(p)}
+    ignored = sorted(p for p in changes if is_test_path(p))
+    scored = pristine.parent / "scored"
+    shutil.copytree(pristine, scored, ignore=shutil.ignore_patterns(".git"))
+    for rel, data in code.items():
+        target = scored / rel
+        if data is None:
+            target.unlink(missing_ok=True)
+        else:
+            target.parent.mkdir(parents=True, exist_ok=True)
+            target.write_bytes(data)
+    patch = unified_diff(pristine, scored, only=sorted(code))
+    with Sandbox(task.image, sandbox) as sb:
+        sb.copy_in(scored)
+        outcomes = run_tests(sb, test_timeout)
+    return outcomes, patch, ignored
+
+
+def run_task(
+    task: Task,
+    model: str | ModelSpec,
+    *,
+    budget: Budget | None = None,
+    context: ContextFile | None = None,
+    strip_context_files: bool = False,
+    agent_config: AgentConfig | None = None,
+    sandbox_config: SandboxConfig | None = None,
+    test_timeout: int = 600,
+    on_step: Callable[[int, str], None] | None = None,
+    keep_trajectory: list | None = None,
+) -> RunResult:
+    """Run one attempt of `task` with `model` and score it. Raises CommitArenaError on fatal setup errors."""
+    spec = model if isinstance(model, ModelSpec) else ModelSpec.parse(model)
+    budget = budget or Budget()
+    sb_cfg = sandbox_config or SandboxConfig()
+    client = ChatClient(spec, budget)
+    ensure_docker()
+    ensure_task_image(task)
+    started = datetime.now(UTC).isoformat(timespec="seconds")
+    t0 = time.monotonic()
+    tmp = Path(tempfile.mkdtemp(prefix="commitarena-run-"))
+    try:
+        pristine = tmp / "pristine"
+        extract_commit(resolve_repo(task.repo), task.parent, pristine)
+        write_files(pristine, task.test_files)
+        if strip_context_files:
+            for n in CONTEXT_NAMES:
+                (pristine / n).unlink(missing_ok=True)
+        system_extra = ""
+        if context:
+            if context.in_repo:
+                (pristine / context.name).write_text(context.content)
+            if context.in_prompt:
+                system_extra = f"# Project instructions ({context.name})\n\n{context.content}"
+        workspace = tmp / "workspace"
+        shutil.copytree(pristine, workspace)
+        init_workspace_git(workspace)
+        agent_tree = tmp / "agent"
+        with Sandbox(task.image, sb_cfg) as sb:
+            sb.copy_in(workspace)
+            outcome = run_agent(client, sb, task.prompt, agent_config, system_extra, on_step)
+            sb.copy_out(agent_tree)
+        if keep_trajectory is not None:
+            keep_trajectory.extend(outcome.trajectory)
+        outcomes, patch, ignored = score_workspace(task, pristine, agent_tree, sb_cfg, test_timeout)
+        passed_set = outcomes.passed()
+        f2p_ok = sum(1 for t in task.fail_to_pass if t in passed_set)
+        p2p_bad = sum(1 for t in task.pass_to_pass if t not in passed_set)
+        return RunResult(
+            task_id=task.id,
+            model=spec.id,
+            passed=outcomes.ok and f2p_ok == len(task.fail_to_pass) and p2p_bad == 0,
+            fail_to_pass_passed=f2p_ok,
+            fail_to_pass_total=len(task.fail_to_pass),
+            pass_to_pass_failed=p2p_bad,
+            pass_to_pass_total=len(task.pass_to_pass),
+            cost_usd=round(outcome.usage.cost_usd, 6),
+            tokens_in=outcome.usage.tokens_in,
+            tokens_out=outcome.usage.tokens_out,
+            steps=outcome.steps,
+            tool_calls=outcome.tool_calls,
+            wall_time_s=round(time.monotonic() - t0, 1),
+            stop_reason=outcome.stop_reason,
+            error=outcome.error,
+            patch=patch,
+            ignored_test_edits=ignored,
+            task_date=task.date,
+            context_file=context.name if context else None,
+            started_at=started,
+        )
+    finally:
+        shutil.rmtree(tmp, ignore_errors=True)
+
+
+def hardware() -> str:
+    mem = ""
+    try:
+        if platform.system() == "Darwin":
+            b = int(subprocess.run(["sysctl", "-n", "hw.memsize"], capture_output=True, text=True).stdout)
+            chip = subprocess.run(
+                ["sysctl", "-n", "machdep.cpu.brand_string"], capture_output=True, text=True
+            ).stdout.strip()
+            mem = f"{chip}, {b // 2**30} GB RAM"
+        elif Path("/proc/meminfo").exists():
+            kb = int(Path("/proc/meminfo").read_text().split()[1])
+            mem = f"{platform.processor() or platform.machine()}, {kb // 2**20} GB RAM"
+    except Exception:  # noqa: BLE001 - best effort only
+        pass
+    return f"{platform.system()} {platform.machine()}" + (f" ({mem})" if mem else "")
+
+
+def run_suite(
+    tasks: list[Task],
+    models: list[ModelSpec],
+    out_dir: Path,
+    *,
+    budget: Budget,
+    context: ContextFile | None = None,
+    strip_context_files: bool = False,
+    agent_config: AgentConfig | None = None,
+    sandbox_config: SandboxConfig | None = None,
+    cutoffs: dict[str, str] | None = None,
+    on_result: Callable[[RunResult], None] | None = None,
+    on_start: Callable[[Task, ModelSpec], None] | None = None,
+    on_step: Callable[[int, str], None] | None = None,
+) -> tuple[list[RunResult], str | None]:
+    """Run every (task, model) pair sequentially. Resumable: pairs already in results.jsonl are skipped.
+
+    Returns (results, abort_message). abort_message is set when the budget stopped the run.
+    """
+    out_dir.mkdir(parents=True, exist_ok=True)
+    (out_dir / "trajectories").mkdir(exist_ok=True)
+    results_path = out_dir / "results.jsonl"
+    done: dict[tuple[str, str], RunResult] = {}
+    if results_path.exists():
+        for line in results_path.read_text().splitlines():
+            if line.strip():
+                r = RunResult.from_dict(json.loads(line))
+                done[(r.task_id, r.model)] = r
+    meta_path = out_dir / "run.json"
+    meta = json.loads(meta_path.read_text()) if meta_path.exists() else {}
+    meta.update(
+        {
+            "tool": "commitarena",
+            "version": __version__,
+            "updated_at": datetime.now(UTC).isoformat(timespec="seconds"),
+            "hardware": hardware(),
+            "models": sorted({*meta.get("models", []), *(m.id for m in models)}),
+            "cutoffs": {**meta.get("cutoffs", {}), **(cutoffs or {})},
+            "agent": {k: v for k, v in asdict(agent_config or AgentConfig()).items() if k != "system_prompt"},
+            "sandbox": asdict(sandbox_config or SandboxConfig()),
+            "context_file": context.name if context else None,
+            "strip_context_files": strip_context_files,
+        }
+    )
+    meta.setdefault("created_at", meta["updated_at"])
+    meta_path.write_text(json.dumps(meta, indent=1))
+    tasks_by_id = {t.id: t for t in tasks}
+    existing_tasks = out_dir / "tasks.json"
+    if existing_tasks.exists():
+        for t in load_tasks(existing_tasks):
+            tasks_by_id.setdefault(t.id, t)
+    save_tasks(existing_tasks, list(tasks_by_id.values()))
+    results = list(done.values())
+    abort = None
+    for model in models:
+        for task in tasks:
+            if (task.id, model.id) in done:
+                continue
+            try:
+                budget.check()
+            except BudgetExceeded as e:
+                return results, str(e)
+            if on_start:
+                on_start(task, model)
+            traj: list = []
+            res = run_task(
+                task, model, budget=budget, context=context, strip_context_files=strip_context_files,
+                agent_config=agent_config,
+                sandbox_config=sandbox_config, on_step=on_step, keep_trajectory=traj,
+            )  # fmt: skip
+            results.append(res)
+            with results_path.open("a") as f:
+                f.write(json.dumps(res.to_dict(), ensure_ascii=False) + "\n")
+            safe = model.id.replace("/", "_").replace(":", "_")
+            (out_dir / "trajectories" / f"{task.id}__{safe}.json").write_text(
+                json.dumps(traj, indent=1, ensure_ascii=False)
+            )
+            if on_result:
+                on_result(res)
+            if res.stop_reason == "budget_exceeded":
+                return results, res.error
+    return results, abort

commitarena/sandbox.py

@@ -0,0 +1,180 @@
+"""Docker sandbox: every command runs in a container started with ``--network none``."""
+
+from __future__ import annotations
+
+import hashlib
+import io
+import shutil
+import subprocess
+import tarfile
+import uuid
+from dataclasses import dataclass
+from pathlib import Path
+
+from .errors import DockerUnavailable
+
+WORKDIR = "/work"
+
+
+def _docker(
+    *args: str, input: bytes | None = None, timeout: float | None = None
+) -> subprocess.CompletedProcess:
+    try:
+        return subprocess.run(["docker", *args], input=input, capture_output=True, timeout=timeout)
+    except FileNotFoundError as e:
+        raise DockerUnavailable(
+            "Docker is not installed (no 'docker' command on PATH). Install Docker Desktop or Docker Engine."
+        ) from e
+
+
+def ensure_docker() -> None:
+    """Raise DockerUnavailable with an actionable message if the daemon is not reachable."""
+    try:
+        cp = _docker("info", "--format", "{{.ServerVersion}}", timeout=20)
+    except subprocess.TimeoutExpired as e:
+        raise DockerUnavailable("Docker did not answer within 20 s. Is Docker Desktop starting up?") from e
+    if cp.returncode != 0:
+        raise DockerUnavailable(
+            "Docker is not running (docker info failed). Start Docker Desktop or the docker service and retry."
+        )
+
+
+@dataclass(frozen=True)
+class SandboxConfig:
+    cpus: float = 2.0
+    memory: str = "2g"
+    pids_limit: int = 512
+    command_timeout: int = 120  # seconds per command
+    network: str = "none"
+
+
+@dataclass(frozen=True)
+class EnvSpec:
+    """How to build the per-repo test image (built once, with network; tasks run without)."""
+
+    python: str = "3.12"
+    setup: str = "pip install --no-cache-dir -e ."
+    extra_packages: str = "pytest==9.2.0"
+
+    def dockerfile(self) -> str:
+        return (
+            f"FROM python:{self.python}-slim\n"
+            "RUN apt-get update && apt-get install -y --no-install-recommends git "
+            "&& rm -rf /var/lib/apt/lists/*\n"
+            f"COPY snapshot/ {WORKDIR}/\n"
+            f"WORKDIR {WORKDIR}\n"
+            f"RUN pip install --no-cache-dir {self.extra_packages} && {self.setup}\n"
+            # Keep installed deps (and the editable .pth pointing at /work); drop the snapshot.
+            f"RUN find {WORKDIR} -mindepth 1 -delete\n"
+            f"ENV PYTHONPATH={WORKDIR}:{WORKDIR}/src PYTHONDONTWRITEBYTECODE=1 PIP_NO_INDEX=1\n"
+        )
+
+    def image_tag(self, slug: str) -> str:
+        h = hashlib.sha256(self.dockerfile().encode()).hexdigest()[:10]
+        return f"commitarena-env-{slug}:{h}"
+
+
+def image_exists(tag: str) -> bool:
+    return _docker("image", "inspect", tag).returncode == 0
+
+
+def build_image(tag: str, env: EnvSpec, snapshot_dir: Path, build_root: Path) -> None:
+    """Build the env image from a snapshot of the repo (network is used only here)."""
+    ensure_docker()
+    ctx = build_root / f"ctx-{uuid.uuid4().hex[:8]}"
+    shutil.copytree(snapshot_dir, ctx / "snapshot", ignore=shutil.ignore_patterns(".git"))
+    (ctx / "Dockerfile").write_text(env.dockerfile())
+    try:
+        cp = _docker("build", "-q", "-t", tag, str(ctx), timeout=1800)
+    finally:
+        shutil.rmtree(ctx, ignore_errors=True)
+    if cp.returncode != 0:
+        tail = cp.stderr.decode(errors="replace")[-2000:]
+        raise DockerUnavailable(f"Building image {tag} failed:\n{tail}")
+
+
+@dataclass
+class ExecResult:
+    exit_code: int
+    output: str
+    timed_out: bool = False
+
+
+def _tar_dir(src: Path) -> bytes:
+    buf = io.BytesIO()
+    with tarfile.open(fileobj=buf, mode="w") as tar:
+        for p in sorted(src.rglob("*")):
+            tar.add(p, arcname=str(p.relative_to(src)), recursive=False)
+    return buf.getvalue()
+
+
+class Sandbox:
+    """A long-lived container with no network. Use as a context manager."""
+
+    def __init__(self, image: str, config: SandboxConfig | None = None):
+        self.image = image
+        self.config = config or SandboxConfig()
+        self.name = f"commitarena-{uuid.uuid4().hex[:12]}"
+        self.started = False
+
+    def __enter__(self) -> Sandbox:
+        self.start()
+        return self
+
+    def __exit__(self, *exc) -> None:
+        self.stop()
+
+    def start(self) -> None:
+        c = self.config
+        cp = _docker(
+            "run", "-d", "--rm", "--name", self.name, "--label", "commitarena=sandbox",
+            "--network", c.network,
+            "--cpus", str(c.cpus), "--memory", c.memory, "--memory-swap", c.memory,
+            "--pids-limit", str(c.pids_limit),
+            "--cap-drop", "ALL", "--security-opt", "no-new-privileges",
+            "-w", WORKDIR, self.image, "sleep", "infinity",
+            timeout=120,
+        )  # fmt: skip
+        if cp.returncode != 0:
+            raise DockerUnavailable(
+                f"Could not start sandbox container: {cp.stderr.decode(errors='replace')}"
+            )
+        self.started = True
+
+    def stop(self) -> None:
+        if self.started:
+            _docker("rm", "-f", self.name, timeout=60)
+            self.started = False
+
+    def copy_in(self, src: Path) -> None:
+        """Copy a host directory into /work. Files end up owned by root (no CAP_DAC_OVERRIDE needed)."""
+        cp = _docker(
+            "exec", "-i", self.name, "tar", "-xf", "-", "-C", WORKDIR, "--no-same-owner",
+            input=_tar_dir(src), timeout=300,
+        )  # fmt: skip
+        if cp.returncode != 0:
+            raise DockerUnavailable(f"Copy into sandbox failed: {cp.stderr.decode(errors='replace')}")
+
+    def copy_out(self, dest: Path) -> None:
+        cp = _docker("exec", self.name, "tar", "-cf", "-", "-C", WORKDIR, ".", timeout=300)
+        if cp.returncode != 0:
+            raise DockerUnavailable(f"Copy out of sandbox failed: {cp.stderr.decode(errors='replace')}")
+        dest.mkdir(parents=True, exist_ok=True)
+        with tarfile.open(fileobj=io.BytesIO(cp.stdout)) as tar:
+            tar.extractall(dest, filter="data")
+
+    def exec(self, command: str, timeout: int | None = None) -> ExecResult:
+        t = int(timeout or self.config.command_timeout)
+        if "\x00" in command:
+            return ExecResult(126, "error: the command contains a null byte; remove it and try again")
+        try:
+            cp = _docker(
+                "exec", self.name, "timeout", "-s", "KILL", str(t), "bash", "-c", "exec 2>&1\n" + command,
+                timeout=t + 30,
+            )  # fmt: skip
+        except subprocess.TimeoutExpired:
+            return ExecResult(124, f"[command timed out after {t}s]", True)
+        out = cp.stdout.decode(errors="replace") + cp.stderr.decode(errors="replace")
+        if cp.returncode in (124, 137) and t:
+            return ExecResult(cp.returncode, out + f"\n[command killed: time limit {t}s]", True)
+        return ExecResult(cp.returncode, out)