benchspec 0.0.1__py3-none-any.whl

benchspec/__init__.py

@@ -0,0 +1,17 @@
+"""benchspec — a benchmark framework for agents.
+
+An eval is one Markdown file: a prompt plus a checklist of prose assertions. Each eval
+runs across the named arms of an eval set — harness × model × effort × environment —
+as parametrized pytest tests, each cell inside its own microVM. Assertions the binder
+can map to a deterministic checker are graded on the host; the rest go to an LLM judge
+that sees only collected evidence. The report is a matrix of pass rates with every
+arm's delta against the set's baseline.
+"""
+
+from __future__ import annotations
+
+__version__ = "0.0.1"
+
+from benchspec.agents import CodingAgent, make_agent
+
+__all__ = ["CodingAgent", "make_agent", "__version__"]

benchspec/__main__.py

@@ -0,0 +1,178 @@
+"""`python -m benchspec <command>` / the `benchspec` console script."""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+from dotenv import find_dotenv, load_dotenv
+
+from benchspec.exit_codes import ExitCode
+from benchspec.reporting import analyze
+from benchspec.runners import run
+from benchspec.sandbox import sandbox
+from benchspec.specs import lint
+from benchspec.specs.schema import SchemaError
+
+
+def _add_root_argument(command_parser: argparse.ArgumentParser) -> None:
+    """Add the shared optional `root` positional to a subcommand parser."""
+    command_parser.add_argument(
+        "root",
+        nargs="?",
+        default=Path("."),
+        type=Path,
+        help="repo root to scan (default: cwd)",
+    )
+
+
+def _add_run_flags(run_parser: argparse.ArgumentParser) -> None:
+    """Add the curated `run` flag vocabulary that forwards to the plugin's options."""
+    run_parser.add_argument("--set", help="eval set to run (--benchspec-set)")
+    run_parser.add_argument("--config", help="config file to load (--benchspec-config)")
+    run_parser.add_argument("--model", help="model for the single arm (--benchspec-model)")
+    run_parser.add_argument("--models", help="comma-separated model sweep (--benchspec-models)")
+    run_parser.add_argument("--harness", help="harness for the single arm (--benchspec-harness)")
+    run_parser.add_argument("--effort", help="reasoning effort (--benchspec-effort)")
+    run_parser.add_argument(
+        "--eval-paths", help="comma-separated search paths (--benchspec-eval-paths)"
+    )
+    run_parser.add_argument("--fail-under", help="pass-rate gate (--benchspec-fail-under)")
+    run_parser.add_argument(
+        "--judge-harness", help="judge harness (--benchspec-judge-harness)"
+    )
+    run_parser.add_argument("--judge-model", help="judge model (--benchspec-judge-model)")
+    run_parser.add_argument("--judge-effort", help="judge effort (--benchspec-judge-effort)")
+    run_parser.add_argument(
+        "--env",
+        action="append",
+        default=[],
+        metavar="KEY=VAL",
+        help="repeatable env var for arms (--benchspec-env); pass --env KEY=VAL per var",
+    )
+
+
+def _split_passthrough(argv: list[str]) -> tuple[list[str], list[str]]:
+    """Split argv at the first standalone `--`, returning the head and verbatim tail.
+
+    Args:
+        argv: The command-line arguments to split.
+
+    Returns:
+        A `(head, tail)` pair — everything before the first standalone `--` and
+        everything after it. The tail is empty when no `--` is present.
+    """
+    if "--" not in argv:
+        return argv, []
+    separator = argv.index("--")
+    return argv[:separator], argv[separator + 1 :]
+
+
+def _run_sandbox_build(args: argparse.Namespace) -> int:
+    """Build the agent-ready snapshot for `root`, mapping host and build errors to exit codes.
+
+    `cli_build` resolves the selected set's backend first, then preflights exactly that backend
+    once — so selected-backend diagnostics are not hidden behind a default-microsandbox preflight,
+    and a future backend can build independently. Error mapping:
+
+    - `SchemaError` (bad config, or a `docker`/unsupported set): propagates to the `main` boundary
+      → USAGE (2). Config error, not a build failure.
+    - `RuntimeError` (host preflight, including microsandbox-not-installed): USAGE (2), surfaced
+      before any provisioning. This branch imports no microsandbox, so a host without the package
+      still exits 2 cleanly rather than raising `ModuleNotFoundError`.
+    - `MicrosandboxError` (a genuine build/provision failure): FINDING (1). Only reachable after
+      preflight passed, which guarantees `import microsandbox` works, so importing the error type
+      here is safe.
+
+    Args:
+        args: The parsed `sandbox:build` namespace, with `root` a `Path`.
+
+    Returns:
+        `ExitCode.SUCCESS` on a built or already-present snapshot, `ExitCode.USAGE` on a
+        preflight or config-schema failure, `ExitCode.FINDING` on a build failure.
+    """
+    root = args.root.resolve()
+
+    try:
+        sandbox.cli_build(root, set_name=args.set, config=args.config)
+    except RuntimeError as error:
+        print(f"error: {error}", file=sys.stderr)
+        return ExitCode.USAGE
+    except SchemaError:
+        raise  # a bad config / unsupported backend is a usage error; let `main` map it to 2
+    except Exception as error:
+        # Reached only after preflight passed, so microsandbox is importable here.
+        from microsandbox.errors import MicrosandboxError
+
+        if isinstance(error, MicrosandboxError):
+            print(f"error: {error}", file=sys.stderr)
+            return ExitCode.FINDING
+        raise
+
+    return ExitCode.SUCCESS
+
+
+def _load_repo_dotenv() -> None:
+    """Load a repo-root `.env` into the environment before any subcommand runs.
+
+    `usecwd=True` walks up from the invocation directory — the same rule the pytest
+    plugin applies — so `lint`, `analyze`, `sandbox:build`, and `run` all see the same
+    credentials. Variables already exported win over `.env` values.
+    """
+    dotenv_path = find_dotenv(usecwd=True)
+    if dotenv_path:
+        load_dotenv(dotenv_path)
+
+
+def main(argv: list[str] | None = None) -> int:
+    """Dispatch the benchspec command-line interface."""
+    _load_repo_dotenv()
+    parser = argparse.ArgumentParser(prog="benchspec")
+    sub = parser.add_subparsers(dest="command", required=True)
+    _add_root_argument(
+        sub.add_parser("lint", help="static checks for eval suites (unjudgeable assertions)")
+    )
+    _add_root_argument(
+        sub.add_parser(
+            "analyze",
+            help="classify assertions deterministic/activation/judge-backed before a run",
+        )
+    )
+    run_parser = sub.add_parser(
+        "run", help="run the eval suite (pytest); pass args after `--` verbatim to pytest"
+    )
+    _add_root_argument(run_parser)
+    _add_run_flags(run_parser)
+    sandbox_build_parser = sub.add_parser(
+        "sandbox:build", help="build the agent-ready sandbox snapshot"
+    )
+    _add_root_argument(sandbox_build_parser)
+    sandbox_build_parser.add_argument(
+        "--set", help="eval set whose sandbox backend + env drive the build"
+    )
+    sandbox_build_parser.add_argument(
+        "--config", help="config file layered over pyproject for set resolution"
+    )
+
+    if argv is None:
+        argv = sys.argv[1:]
+    head, passthrough = _split_passthrough(argv)
+    args = parser.parse_args(head)
+    args.passthrough = passthrough
+
+    dispatch = {
+        "lint": lambda parsed: lint.run(parsed.root.resolve()),
+        "analyze": lambda parsed: analyze.run(parsed.root.resolve()),
+        "run": run.run,
+        "sandbox:build": _run_sandbox_build,
+    }
+    try:
+        return dispatch[args.command](args)
+    except SchemaError as error:
+        print(f"error: {error}", file=sys.stderr)
+        return ExitCode.USAGE
+
+
+if __name__ == "__main__":
+    sys.exit(main())

benchspec/agents/__init__.py

@@ -0,0 +1,106 @@
+"""Agents for benchspec.
+
+`sandbox.py` talks only to the `CodingAgent` interface and to the factory/preflight
+helpers here — never to a concrete agent. The active agent is resolved via
+`resolve_agent_name`'s precedence chain (--benchspec-agent flag > BENCHSPEC_AGENT env >
+[tool.benchspec] agent > claude-code default), with `BENCHSPEC_AGENT` being the normalized
+handoff that `make_agent()` reads. Unknown values fail loudly. Adding a third agent is
+additive: implement the protocol, register it in `_REGISTRY`.
+"""
+
+from __future__ import annotations
+
+import os
+
+from benchspec.agents.base import AgentCapabilities, CodingAgent
+from benchspec.agents.claude import ClaudeCodeAgent
+from benchspec.agents.codex import CodexAgent
+from benchspec.agents.opencode import OpenCodeAgent
+
+__all__ = [
+    "AgentCapabilities",
+    "CodingAgent",
+    "ClaudeCodeAgent",
+    "CodexAgent",
+    "OpenCodeAgent",
+    "agent_class",
+    "make_agent",
+    "credential_preflight_error",
+    "resolve_agent_name",
+    "known_harnesses",
+]
+
+_REGISTRY: dict[str, type] = {
+    "claude-code": ClaudeCodeAgent,
+    "codex": CodexAgent,
+    "opencode": OpenCodeAgent,
+}
+
+_DEFAULT_AGENT = "claude-code"
+
+
+def known_harnesses() -> frozenset[str]:
+    """Registered agent names — the authoritative set of valid harness values."""
+    return frozenset(_REGISTRY)
+
+
+def resolve_agent_name(flag: str | None = None, pyproject: str | None = None) -> str:
+    """The agent for this run: --benchspec-agent > BENCHSPEC_AGENT > [tool.benchspec].
+
+    agent > claude-code. An unknown value fails naming the source it came from, so a
+    typo dies at startup with a fixable message instead of mid-run.
+    """
+    sources = (
+        ("--benchspec-agent", flag),
+        ("BENCHSPEC_AGENT", os.environ.get("BENCHSPEC_AGENT")),
+        ("[tool.benchspec] agent", pyproject),
+    )
+    for source_name, value in sources:
+        if value:
+            if value not in _REGISTRY:
+                valid_agents = sorted(_REGISTRY)
+                raise RuntimeError(
+                    f"{source_name}={value!r} is not a known agent; valid: {valid_agents}"
+                )
+            return value
+    return _DEFAULT_AGENT
+
+
+def _selected_agent_class() -> type:
+    """Provide the selected agent class helper."""
+    return _REGISTRY[resolve_agent_name()]
+
+
+def agent_class(harness: str) -> type:
+    """The registered adapter class for a harness name.
+
+    Judge-mode dispatch looks the class up, then binds an instance to the host
+    environment via `for_host()` — the judge uses the host's own credentials, so no
+    `from_env()` credential read is involved. An unknown `harness` raises.
+    """
+    if harness not in _REGISTRY:
+        raise RuntimeError(
+            f"harness {harness!r} is not a known agent; valid: {sorted(_REGISTRY)}"
+        )
+    return _REGISTRY[harness]
+
+
+def make_agent(harness: str | None = None) -> CodingAgent:
+    """The agent for a run or a single arm.
+
+    With no `harness`, reads the run-level agent from `BENCHSPEC_AGENT` (the plugin
+    normalizes the precedence chain at configure time). With an explicit `harness` (an
+    arm's `harness`), builds THAT agent so one run's columns can span harnesses. An
+    unknown `harness` raises.
+    """
+    if harness is not None:
+        return agent_class(harness).from_env()
+    return _selected_agent_class().from_env()
+
+
+def credential_preflight_error() -> str | None:
+    """None if a usable credential is configured for the selected agent, else a.
+
+    remediation message for preflight.
+    """
+    return _selected_agent_class().credential_error()

benchspec/agents/base.py

@@ -0,0 +1,305 @@
+"""The agent interface.
+
+A `CodingAgent` hides everything agent-specific behind one boundary: where its home
+lives in the guest, how to provision the CLI into a microVM (the cached step), which
+secrets it needs, how to stage local skills, how to build its headless command, and how
+to parse its output. `sandbox.py` drives a live sandbox through this interface and never
+names a concrete agent; adding a second agent is additive, not a refactor.
+
+One adapter per harness, transport-blind: the adapter builds commands and parses
+output, and a `benchspec.orchestration.environments.ExecutionEnv` decides where the
+process runs —
+`GuestSandbox` for task arms (`invoke`), `Host` for grading (`judge`). Sandbox-vs-host
+is a parameter of the call, not a code path baked into each harness.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import re
+import subprocess
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Protocol, runtime_checkable
+
+from benchspec.orchestration.results import RunResult
+
+if TYPE_CHECKING:
+    from benchspec.grading.judges.config import JudgeConfig
+    from benchspec.orchestration.environments import ExecutionEnv
+    from benchspec.sandbox.backend import SandboxBackend
+
+# The agent-neutral home every per-cell `setup.sh` copies skills into. Each agent
+# symlinks its own load dir here once at provision, so the install path is identical
+# across agents and the per-agent load dir is the only agent-specific fact.
+FIXED_SKILLS_HOME = "/home/benchspec/skills"
+
+# Matches the first dotted-numeric token in `--version` output, e.g. the "1.2.3" in
+# both "claude-code 1.2.3" and "codex-cli 0.144.1". Shared by every guest-version parse
+# so adapters never hand-roll their own extraction.
+_VERSION_TOKEN_RE = re.compile(r"\d+(?:\.\d+)+")
+
+# The guest `--version` probe runs before the task on every sample. A wedged guest
+# command must not block the run forever, so the await is bounded and a timeout becomes
+# an explained-unavailable result like any other probe failure.
+GUEST_VERSION_PROBE_TIMEOUT_SECONDS = 30.0
+
+
+def _parse_version_token(output: str) -> str | None:
+    """Extract a dotted version token (e.g. "1.2.3") from raw `--version` output."""
+    match = _VERSION_TOKEN_RE.search(output)
+    return match.group(0) if match else None
+
+
+async def probe_guest_version(
+    backend: SandboxBackend, sandbox: object, agent: object
+) -> tuple[str | None, str | None]:
+    """Measure the task-harness binary's version inside the live guest sandbox.
+
+    Runs `<agent.agent_bin> --version` through the backend's guest command seam
+    (`backend.guest_shell`) against the already-booted `sandbox` instance. This is the
+    guest task-harness probe: it reads the binary actually installed in the selected
+    snapshot, never the host binding (`for_host()`) or the pinned install selector
+    (`agent.version()`, e.g. `"latest"`).
+
+    Never raises: any failure to reach the guest, or to parse a version out of what it
+    returns, is reported as an explained `(None, error)` pair rather than an
+    unexplained null (ground rule: explained unavailable).
+
+    Args:
+        backend: The resolved `SandboxBackend` driving this arm's sandbox.
+        sandbox: The live sandbox instance for the running arm session.
+        agent: The `CodingAgent` whose `agent_bin` is probed.
+
+    Returns:
+        `(version, None)` on success, or `(None, error)` describing why the probe
+        could not produce a version.
+    """
+    script = f"{agent.agent_bin} --version"
+    try:
+        # Bounded so a wedged guest command surfaces as unavailable instead of hanging
+        # every sample before its task runs.
+        output = await asyncio.wait_for(
+            backend.guest_shell(sandbox, agent, script),
+            timeout=GUEST_VERSION_PROBE_TIMEOUT_SECONDS,
+        )
+    except TimeoutError:
+        return None, f"guest version probe timed out after {GUEST_VERSION_PROBE_TIMEOUT_SECONDS}s"
+    except Exception as error:
+        # Broad on purpose: this probe's contract is "never raise" (see docstring), and
+        # guest_shell's own concrete failure modes vary by backend.
+        return None, f"guest_shell raised {type(error).__name__}: {error}"
+    if output is None:
+        return None, f"guest_shell returned no output for `{script}`"
+    version = _parse_version_token(output)
+    if version is None:
+        return None, f"could not parse a version from guest output: {output.strip()!r}"
+    return version, None
+
+
+class BaseAgent:
+    """Store base agent data."""
+
+    # The binary THIS instance runs. An instance is bound to one execution
+    # environment: the default binding is the guest install path (task arms);
+    # for_host() rebinds to the name PATH resolves on the host (judge mode).
+    agent_bin: str
+
+    def provision_script(self: object) -> str:
+        """The fully-resolved commands this instance runs to install the CLI in the guest.
+
+        Concrete task adapters override it, baking in any instance state (e.g. a pinned
+        version). `provision()` runs exactly this, and `install_fingerprint()` hashes it — so
+        every install-affecting input is captured in the cache key structurally, with nothing
+        to fold by hand. Empty on the base (host/judge agents install nothing in a guest).
+        """
+        return ""
+
+    def binary_version(self: object) -> str | None:
+        """Best-effort `agent_bin --version` probe — never raises, never fails the run."""
+        try:
+            proc = subprocess.run(
+                [self.agent_bin, "--version"], capture_output=True, text=True, timeout=10
+            )
+        except (FileNotFoundError, OSError, subprocess.TimeoutExpired):
+            return None
+        if proc.returncode != 0:
+            return None
+        return proc.stdout.strip() or None
+
+    def install_fingerprint(self: object) -> str:
+        """Cache-key fingerprint of the CLI install: a hash of the resolved provision script.
+
+        `provision_script()` is the single source of truth for what installs the CLI, so
+        changing the installer (a new revision, package list, pinned version, or bootstrap
+        commands) rebuilds the snapshot. The agent version also appears directly in the
+        snapshot name, so a version bump rebuilds even for an adapter whose installer does not
+        embed the version.
+        """
+        return hashlib.sha256(self.provision_script().encode()).hexdigest()[:12]
+
+    def bridge_skills_home_script(self: object) -> str:
+        """Bridge skills home script."""
+        skill_dir = self.skill_load_dir
+        parent = skill_dir.rsplit("/", 1)[0]
+        return (
+            f"mkdir -p {FIXED_SKILLS_HOME} && "
+            f"mkdir -p {parent} && rm -rf {skill_dir} && "
+            f"ln -s {FIXED_SKILLS_HOME} {skill_dir}"
+        )
+
+    def cell_env(self: object, *, arm: str, model: str, eval_set: str = "") -> dict:
+        """BENCHSPEC_* are informational for setup.sh — they do NOT route the task model.
+
+        (that goes through arm.model). BENCHSPEC_SET names the explicitly-selected set
+        (--benchspec-set / make evals SET=) so setup.sh can branch on it; empty when the
+        run falls back to the pyproject default-set.
+        """
+        return {
+            **self.guest_env(),
+            "BENCHSPEC_ARM": arm,
+            "BENCHSPEC_MODEL": model,
+            "BENCHSPEC_HARNESS": self.id,
+            "BENCHSPEC_SET": eval_set,
+        }
+
+
+@dataclass(frozen=True)
+class AgentCapabilities:
+    """What the harness can honestly do with this agent.
+
+    Typed, not a dict: every field here has a consumer in the harness, and an
+    unknown field is a type error rather than silently ignored documentation.
+    Effort is deliberately not a capability: every driver forwards it verbatim and
+    the agent's CLI is the validator, the same way model names are handled.
+    """
+
+    multi_turn: bool  # honors resume_session_id (session chaining across turns)
+    token_split: bool  # reports input/output token split (enables cost estimates)
+
+
+@runtime_checkable
+class CodingAgent(Protocol):
+    """Define the agent interface."""
+
+    id: str  # snapshot-cache key + report label
+    guest_home: str  # the agent's HOME inside the guest (where skills are staged, runs cwd)
+    skill_load_dir: str  # absolute guest path the agent loads skills from
+    agent_bin: str  # the binary this instance runs (guest install path; for_host() rebinds)
+    capabilities: AgentCapabilities
+
+    def version(self: object) -> str:
+        """Return the agent CLI version string."""
+        ...
+
+    def provision_script(self: object) -> str:
+        """Return the fully-resolved commands that install the CLI in the guest."""
+        ...
+
+    def install_fingerprint(self: object) -> str:
+        """Return the CLI install fingerprint (a hash of `provision_script()`)."""
+        ...
+
+    def bridge_skills_home_script(
+        self: object,
+    ) -> str:
+        """Bridge skills home script."""
+        ...
+
+    def cell_env(self: object, *, arm: str, model: str, eval_set: str = "") -> dict:
+        """Return per-cell environment variables for an arm run."""
+        ...
+
+    def artifact_dirs(
+        self: object,
+    ) -> list[str]:
+        """Return guest directories that may contain agent-authored artifacts."""
+        ...
+
+    #   artifacts (e.g. ~/.claude/skills). These live in the VM, NOT the workdir mount, so
+    #   the session snapshots them and merges the agent-authored files (diffed against the
+    #   staged baseline) into the judge's facts. Each agent scaffolds skills differently, so
+    #   each owns its answer; return [] for an agent that writes only to the workdir.
+    def secrets(self: object) -> list:
+        """Return secret values that must be redacted from logs."""
+        ...
+
+    def guest_env(self: object) -> dict:
+        """Return environment variables passed to guest agent commands."""
+        ...
+
+    def build_command(
+        self: object,
+        prompt: str,
+        *,
+        plugin_dir: str | None,
+        model: str,
+        effort: str,
+        resume_session_id: str | None,
+        detect_skill: str | None,
+        harness_args: list[str] | None = None,
+    ) -> list[str]:
+        """Build the guest command used to invoke the agent."""
+        ...
+
+    async def provision(self: object, sandbox: object) -> None:
+        """Install the agent CLI and credentials inside the guest."""
+        ...
+
+    async def stage_project_assets(self: object, sandbox: object, project_mount: str) -> None:
+        """Copy project-local assets needed by the guest agent."""
+        ...
+
+    async def invoke(
+        self: object,
+        sandbox: object,
+        prompt: str,
+        *,
+        eval_id: str,
+        config: str,
+        workdir: str,
+        plugin_dir: str | None,
+        model: str,
+        effort: str,
+        resume_session_id: str | None,
+        detect_skill: str | None,
+        harness_args: list[str] | None = None,
+        extra_env: dict | None = None,
+    ) -> RunResult:
+        """Run one prompt through the agent inside the guest."""
+        ...
+
+    async def judge(
+        self: object,
+        prompt: str,
+        config: JudgeConfig,
+        *,
+        env: ExecutionEnv | None = None,
+    ) -> str:
+        """Grade a judge prompt in `env` (default: a fresh Host process).
+
+        Returns the {"result": "<judge-json-string>"} envelope judge.py parses.
+        Raises RuntimeError for the harness's infra-failure shapes.
+        """
+        ...
+
+    @classmethod
+    def for_host(cls: object) -> CodingAgent:
+        """An instance bound to the host environment: agent_bin resolves from PATH."""
+        ...
+
+    def binary_version(self: object) -> str | None:
+        """Return this instance's CLI version, or None on any failure (best effort)."""
+        ...
+
+    def detect_dispatch(self: object, line: str, skill_name: str | None) -> bool:
+        """Return whether one stream line shows a skill dispatch."""
+        ...
+
+    def detect_fired(self: object, lines: object, skill_name: str) -> bool:
+        """Return whether stream lines show the expected skill firing."""
+        ...
+
+    def streamed_activity(self: object, lines: object) -> bool:
+        """Return whether streamed output shows meaningful agent activity."""
+        ...