verifiers 0.2.1.dev47__tar.gz → 0.2.1.dev49__tar.gz

{verifiers-0.2.1.dev47 → verifiers-0.2.1.dev49}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.1.dev47
+Version: 0.2.1.dev49
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers
@@ -35,7 +35,7 @@ Requires-Dist: numpy
 Requires-Dist: openai-agents>=0.0.7
 Requires-Dist: openai>=1.108.1
 Requires-Dist: prime-pydantic-config[toml]>=0.3.0.dev86
-Requires-Dist: prime-sandboxes>=0.2.27
+Requires-Dist: prime-sandboxes>=0.2.31
 Requires-Dist: prime-tunnel>=0.1.8
 Requires-Dist: pydantic>=2.11.9
 Requires-Dist: pyzmq>=27.1.0

{verifiers-0.2.1.dev47 → verifiers-0.2.1.dev49}/pyproject.toml

@@ -35,7 +35,7 @@ dependencies = [
     "openai>=1.108.1",
     "openai-agents>=0.0.7",
     "prime-tunnel>=0.1.8",
-    "prime-sandboxes>=0.2.27",
+    "prime-sandboxes>=0.2.31",
     "pydantic>=2.11.9",
     "requests",
     "rich",

{verifiers-0.2.1.dev47 → verifiers-0.2.1.dev49}/tests/conftest.py

@@ -39,6 +39,12 @@ from verifiers.types import (
 from verifiers.utils.save_utils import state_to_output
 
 
+def pytest_configure(config) -> None:
+    config.addinivalue_line(
+        "markers", "claude_code: v1 e2e cases on the claude-code harness"
+    )
+
+
 @pytest.fixture
 def basic_parser():
     """Return a basic Parser instance."""

{verifiers-0.2.1.dev47 → verifiers-0.2.1.dev49}/tests/v1/conftest.py

@@ -20,10 +20,11 @@ Every matrix value carries a pytest mark, so subsets select with `-m`:
     uv run pytest tests/v1 -n auto -m modal                       # only modal (needs local setup)
 
 Marks: runtimes `subprocess` / `docker` / `prime` / `modal`, placement `colocated`,
-harnesses `null` / `default` / `rlm` / `kimi_code` / `codex`. A mark is applied per axis, so it
-selects every case touching that value on ANY axis; for one exact combination use `-k` on the test
-id (e.g. `-k "harness-in-docker-with-tool-in-subprocess"`). prime/modal provision real remote
-sandboxes (slow, infra-flaky, need setup), so they're local-only — CI runs `-m "not prime and not modal"`.
+harnesses `null` / `default` / `rlm` / `kimi_code` / `codex` / `claude_code`. A mark is applied per
+axis, so it selects every case touching that value on ANY axis; for one exact combination use `-k`
+on the test id (e.g. `-k "harness-in-docker-with-tool-in-subprocess"`). prime/modal provision real
+remote sandboxes (slow, infra-flaky, need setup), so they're local-only — CI runs
+`-m "not prime and not modal"`.
 """
 
 import os
@@ -104,10 +105,10 @@ def tool_runtime(request) -> dict:
 
 
 # Harnesses, composed with the runtime fixtures, each carrying its harness mark (`-m default`, ...).
-# Built-ins are bundled in the `harnesses` package; the agent CLIs (`rlm` / `kimi-code` / `codex`)
-# install their dependencies at rollout. `compact` (an example harness) and `terminus-2` (drives
-# the host tmux) are excluded. `test_agentic` skips `null` (a chat loop with no shell);
-# `test_single_turn` skips `codex` (a coding agent, unreliable on a no-op echo).
+# Built-ins are bundled in the `harnesses` package; the agent CLIs (`rlm` / `kimi-code` / `codex` /
+# `claude-code`) install their dependencies at rollout. `compact` (an example harness) and
+# `terminus-2` (drives the host tmux) are excluded. `test_agentic` skips `null` (a chat loop with no
+# shell); `test_single_turn` skips the coding agents, which are unreliable on a no-op echo.
 @pytest.fixture(
     params=[
         pytest.param("null", marks=pytest.mark.null, id="null"),
@@ -115,6 +116,7 @@ def tool_runtime(request) -> dict:
         pytest.param("rlm", marks=pytest.mark.rlm, id="rlm"),
         pytest.param("kimi-code", marks=pytest.mark.kimi_code, id="kimi-code"),
         pytest.param("codex", marks=pytest.mark.codex, id="codex"),
+        pytest.param("claude-code", marks=pytest.mark.claude_code, id="claude-code"),
     ]
 )
 def harness(request) -> str:

{verifiers-0.2.1.dev47 → verifiers-0.2.1.dev49}/tests/v1/test_e2e.py

@@ -6,8 +6,8 @@ import pytest
 @pytest.mark.e2e
 async def test_single_turn(run_v1, harness, harness_runtime, tmp_path):
     """Single-turn (echo a short phrase back)."""
-    if harness == "codex":
-        pytest.skip("codex is a coding agent, not reliable on a no-op echo chat task")
+    if harness in {"codex", "claude-code"}:
+        pytest.skip("coding agents are not reliable on a no-op echo chat task")
     (trace,) = await run_v1(
         "echo-v1",
         harness=harness,

{verifiers-0.2.1.dev47 → verifiers-0.2.1.dev49}/verifiers/v1/cli/dashboard/eval.py

@@ -41,6 +41,7 @@ _LABEL_WIDTH = len("timeouts")
 _STYLE = {
     "pending": "dim",
     "boot": "orange3",
+    "build": "dark_orange",
     "setup": "yellow",
     "running": "cyan",
     "finalize": "magenta",
@@ -51,6 +52,7 @@ _STYLE = {
 _MARK_LABEL = {
     "pending": "pending",
     "boot": "boot",
+    "build": "build",
     "setup": "setup",
     "running": "rollout",
     "finalize": "finalize",
@@ -445,6 +447,12 @@ def Rows(groups: list[list[Rollout]], now: float, runtime_type: str) -> Table:
                         stop = f"{stop} (truncated)".strip()
             else:
                 state, result, stop = rollout.phase, "", ""
+                # A boot stuck on a first-use platform image build reads differently from a
+                # normal boot — it can sit here for ~10 minutes (prime runtime only).
+                if state == Phase.BOOT and (
+                    getattr(t.runtime, "image_cached", None) is False
+                ):
+                    state = "build"
             runtime_id = t.runtime.id if t.runtime is not None else None
             runtime = f"{runtime_type}({runtime_id})" if runtime_id else runtime_type
             turns = t.num_turns

{verifiers-0.2.1.dev47 → verifiers-0.2.1.dev49}/verifiers/v1/harnesses/__init__.py

@@ -1,3 +1,7 @@
+from verifiers.v1.harnesses.claude_code import (
+    ClaudeCodeHarness,
+    ClaudeCodeHarnessConfig,
+)
 from verifiers.v1.harnesses.codex import CodexHarness, CodexHarnessConfig
 from verifiers.v1.harnesses.default import DefaultHarness, DefaultHarnessConfig
 from verifiers.v1.harnesses.kimi_code import KimiCodeHarness, KimiCodeHarnessConfig
@@ -11,6 +15,8 @@ from verifiers.v1.harnesses.rlm import RLMHarness, RLMHarnessConfig
 from verifiers.v1.harnesses.terminus_2 import Terminus2Harness, Terminus2HarnessConfig
 
 __all__ = [
+    "ClaudeCodeHarness",
+    "ClaudeCodeHarnessConfig",
     "CodexHarness",
     "CodexHarnessConfig",
     "DefaultHarness",

verifiers-0.2.1.dev49/verifiers/v1/harnesses/claude_code/__init__.py

@@ -0,0 +1,6 @@
+from verifiers.v1.harnesses.claude_code.harness import (
+    ClaudeCodeHarness,
+    ClaudeCodeHarnessConfig,
+)
+
+__all__ = ["ClaudeCodeHarness", "ClaudeCodeHarnessConfig"]

verifiers-0.2.1.dev49/verifiers/v1/harnesses/claude_code/harness.py

@@ -0,0 +1,102 @@
+"""Run Claude Code against interception as an Anthropic Messages client."""
+
+import json
+import shlex
+
+from pydantic import Field
+
+from verifiers.v1.clients import ModelContext
+from verifiers.v1.harness import Harness, HarnessConfig
+from verifiers.v1.runtimes import ProgramResult, Runtime
+from verifiers.v1.trace import Trace
+
+CLAUDE_HOME = "/tmp/vf-claude-code-{version}"
+CLAUDE_BIN = f"{CLAUDE_HOME}/.local/bin/claude"
+CLAUDE_CONFIG_DIR = ".vf-claude"
+INSTALL = """
+set -e
+command -v curl >/dev/null || (apt-get update -qq && apt-get install -y -qq curl ca-certificates >/dev/null)
+curl -fsSL https://claude.ai/install.sh | HOME={home} bash -s {version}
+"""
+
+
+class ClaudeCodeHarnessConfig(HarnessConfig):
+    version: str = Field(default="2.1.207", pattern=r"^[A-Za-z0-9._+-]+$")
+    """Claude Code release to install; pinned for reproducibility."""
+
+
+class ClaudeCodeHarness(Harness[ClaudeCodeHarnessConfig]):
+    APPENDS_SYSTEM_PROMPT = True
+    SUPPORTS_MCP = True
+    # images would require streaming inputs
+    SUPPORTS_MESSAGE_PROMPT = False
+
+    async def setup(self, runtime: Runtime) -> None:
+        home = CLAUDE_HOME.format(version=self.config.version)
+        binary = CLAUDE_BIN.format(version=self.config.version)
+        script = INSTALL.format(version=self.config.version, home=home)
+        # Cache the pinned binary across local rollouts; Linux has flock, macOS has lockf.
+        install = shlex.quote(f"[ -x {binary} ] || ({script})")
+        guarded = (
+            f"mkdir -p {home} && "
+            f'"$(command -v flock || command -v lockf)" {home}/install.lock '
+            f"bash -o pipefail -c {install}"
+        )
+        result = await runtime.run(["sh", "-c", guarded], self.config.resolved_env)
+        if result.exit_code != 0:
+            detail = (result.stderr or result.stdout).strip()[-500:]
+            raise RuntimeError(f"Claude Code install failed: {detail}")
+
+    async def launch(
+        self,
+        ctx: ModelContext,
+        trace: Trace,
+        runtime: Runtime,
+        endpoint: str,
+        secret: str,
+        mcp_urls: dict[str, str],
+    ) -> ProgramResult:
+        system_prompt, instruction = self.resolve_prompt(trace.task.data)
+        if ctx.client.base_url == "https://api.pinference.ai/api/v1":
+            # remove the /v1 from pinference
+            ctx.client.base_url = ctx.client.base_url.removesuffix("/v1")
+        env = {
+            **self.config.resolved_env,
+            # Claude appends /v1/messages; give it the interception root, not the model endpoint.
+            "ANTHROPIC_BASE_URL": endpoint.removesuffix("/v1"),
+            "ANTHROPIC_API_KEY": secret,
+            "CLAUDE_CONFIG_DIR": CLAUDE_CONFIG_DIR,
+            "DISABLE_AUTOUPDATER": "1",
+            "IS_SANDBOX": "1",
+        }
+        argv = [
+            CLAUDE_BIN.format(version=self.config.version),
+            "--print",
+            "--bare",
+            "--dangerously-skip-permissions",
+            "--no-session-persistence",
+            "--model",
+            ctx.model,
+        ]
+        if system_prompt:
+            argv += ["--append-system-prompt", system_prompt]
+        argv += [
+            arg
+            for tool in self.config.disabled_tools or []
+            for arg in ("--disallowedTools", tool)
+        ]
+        mcp = {
+            "mcpServers": {
+                name: {"type": "http", "url": url} for name, url in mcp_urls.items()
+            }
+        }
+        mcp_path = f"{CLAUDE_CONFIG_DIR}/mcp.json"
+        await runtime.write(mcp_path, json.dumps(mcp).encode())
+        argv += [
+            "--mcp-config",
+            mcp_path,
+            "--strict-mcp-config",
+            "--",
+            instruction or "",
+        ]
+        return await runtime.run_program(argv, env)

{verifiers-0.2.1.dev47 → verifiers-0.2.1.dev49}/verifiers/v1/runtimes/prime.py

@@ -37,6 +37,10 @@ MAX_LIFETIME = 24 * 60 * 60
 class PrimeConfig(BaseConfig):
     type: Literal["prime"] = "prime"
     image: str = "python:3.11-slim"
+    """Docker image to run. Any pullable ref works: on the first use of an image, the
+    platform auto-builds what the sandbox needs from it (a VM image for `vm` sandboxes,
+    ~10 minutes) and caches the result, so later sandboxes on the same ref start in
+    seconds."""
     workdir: str = "/app"
     network_access: bool = True
     vm: bool = False
@@ -74,7 +78,9 @@ class PrimeConfig(BaseConfig):
 
 
 class PrimeRuntimeInfo(PrimeConfig, BaseRuntimeInfo):
-    pass
+    image_cached: bool | None = None
+    """Whether the platform already had the image at create (None until then). False means
+    a first-use auto-build ran while this sandbox waited to start."""
 
 
 class PrimeRuntime(Runtime):
@@ -134,6 +140,19 @@ class PrimeRuntime(Runtime):
                     )
                 )
             self.info.id = sandbox.id
+            # The create response says whether the platform already has the image:
+            # `pending_image_build_id` set means a first-use auto-build is running and the
+            # sandbox stays PENDING until it finishes (`wait_for_creation` gives that phase
+            # its own budget, separate from the normal boot attempts).
+            self.info.image_cached = sandbox.pending_image_build_id is None
+            if not self.info.image_cached:
+                logger.warning(
+                    "prime: image %s isn't cached on the platform - auto-building it "
+                    "(sandbox %s waits for the build; first use of an image can take "
+                    "~10 minutes, later runs start in seconds)",
+                    self.config.image,
+                    self.info.id,
+                )
             await self._client.wait_for_creation(self.info.id)
             logger.info(
                 "prime: sandbox %s up (image=%s)", self.info.id, self.config.image