verifiers 0.2.1.dev48__tar.gz → 0.2.1.dev50__tar.gz

{verifiers-0.2.1.dev48 → verifiers-0.2.1.dev50}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.1.dev48
+Version: 0.2.1.dev50
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.2.1.dev48 → verifiers-0.2.1.dev50}/tests/conftest.py

@@ -39,6 +39,12 @@ from verifiers.types import (
 from verifiers.utils.save_utils import state_to_output
 
 
+def pytest_configure(config) -> None:
+    config.addinivalue_line(
+        "markers", "claude_code: v1 e2e cases on the claude-code harness"
+    )
+
+
 @pytest.fixture
 def basic_parser():
     """Return a basic Parser instance."""

{verifiers-0.2.1.dev48 → verifiers-0.2.1.dev50}/tests/v1/conftest.py

@@ -20,10 +20,11 @@ Every matrix value carries a pytest mark, so subsets select with `-m`:
     uv run pytest tests/v1 -n auto -m modal                       # only modal (needs local setup)
 
 Marks: runtimes `subprocess` / `docker` / `prime` / `modal`, placement `colocated`,
-harnesses `null` / `default` / `rlm` / `kimi_code` / `codex`. A mark is applied per axis, so it
-selects every case touching that value on ANY axis; for one exact combination use `-k` on the test
-id (e.g. `-k "harness-in-docker-with-tool-in-subprocess"`). prime/modal provision real remote
-sandboxes (slow, infra-flaky, need setup), so they're local-only — CI runs `-m "not prime and not modal"`.
+harnesses `null` / `default` / `rlm` / `kimi_code` / `codex` / `claude_code`. A mark is applied per
+axis, so it selects every case touching that value on ANY axis; for one exact combination use `-k`
+on the test id (e.g. `-k "harness-in-docker-with-tool-in-subprocess"`). prime/modal provision real
+remote sandboxes (slow, infra-flaky, need setup), so they're local-only — CI runs
+`-m "not prime and not modal"`.
 """
 
 import os
@@ -104,10 +105,10 @@ def tool_runtime(request) -> dict:
 
 
 # Harnesses, composed with the runtime fixtures, each carrying its harness mark (`-m default`, ...).
-# Built-ins are bundled in the `harnesses` package; the agent CLIs (`rlm` / `kimi-code` / `codex`)
-# install their dependencies at rollout. `compact` (an example harness) and `terminus-2` (drives
-# the host tmux) are excluded. `test_agentic` skips `null` (a chat loop with no shell);
-# `test_single_turn` skips `codex` (a coding agent, unreliable on a no-op echo).
+# Built-ins are bundled in the `harnesses` package; the agent CLIs (`rlm` / `kimi-code` / `codex` /
+# `claude-code`) install their dependencies at rollout. `compact` (an example harness) and
+# `terminus-2` (drives the host tmux) are excluded. `test_agentic` skips `null` (a chat loop with no
+# shell); `test_single_turn` skips the coding agents, which are unreliable on a no-op echo.
 @pytest.fixture(
     params=[
         pytest.param("null", marks=pytest.mark.null, id="null"),
@@ -115,6 +116,7 @@ def tool_runtime(request) -> dict:
         pytest.param("rlm", marks=pytest.mark.rlm, id="rlm"),
         pytest.param("kimi-code", marks=pytest.mark.kimi_code, id="kimi-code"),
         pytest.param("codex", marks=pytest.mark.codex, id="codex"),
+        pytest.param("claude-code", marks=pytest.mark.claude_code, id="claude-code"),
     ]
 )
 def harness(request) -> str:

{verifiers-0.2.1.dev48 → verifiers-0.2.1.dev50}/tests/v1/test_e2e.py

@@ -6,8 +6,8 @@ import pytest
 @pytest.mark.e2e
 async def test_single_turn(run_v1, harness, harness_runtime, tmp_path):
     """Single-turn (echo a short phrase back)."""
-    if harness == "codex":
-        pytest.skip("codex is a coding agent, not reliable on a no-op echo chat task")
+    if harness in {"codex", "claude-code"}:
+        pytest.skip("coding agents are not reliable on a no-op echo chat task")
     (trace,) = await run_v1(
         "echo-v1",
         harness=harness,

{verifiers-0.2.1.dev48 → verifiers-0.2.1.dev50}/verifiers/v1/harnesses/__init__.py

@@ -1,3 +1,7 @@
+from verifiers.v1.harnesses.claude_code import (
+    ClaudeCodeHarness,
+    ClaudeCodeHarnessConfig,
+)
 from verifiers.v1.harnesses.codex import CodexHarness, CodexHarnessConfig
 from verifiers.v1.harnesses.default import DefaultHarness, DefaultHarnessConfig
 from verifiers.v1.harnesses.kimi_code import KimiCodeHarness, KimiCodeHarnessConfig
@@ -11,6 +15,8 @@ from verifiers.v1.harnesses.rlm import RLMHarness, RLMHarnessConfig
 from verifiers.v1.harnesses.terminus_2 import Terminus2Harness, Terminus2HarnessConfig
 
 __all__ = [
+    "ClaudeCodeHarness",
+    "ClaudeCodeHarnessConfig",
     "CodexHarness",
     "CodexHarnessConfig",
     "DefaultHarness",

verifiers-0.2.1.dev50/verifiers/v1/harnesses/claude_code/__init__.py

@@ -0,0 +1,6 @@
+from verifiers.v1.harnesses.claude_code.harness import (
+    ClaudeCodeHarness,
+    ClaudeCodeHarnessConfig,
+)
+
+__all__ = ["ClaudeCodeHarness", "ClaudeCodeHarnessConfig"]

verifiers-0.2.1.dev50/verifiers/v1/harnesses/claude_code/harness.py

@@ -0,0 +1,102 @@
+"""Run Claude Code against interception as an Anthropic Messages client."""
+
+import json
+import shlex
+
+from pydantic import Field
+
+from verifiers.v1.clients import ModelContext
+from verifiers.v1.harness import Harness, HarnessConfig
+from verifiers.v1.runtimes import ProgramResult, Runtime
+from verifiers.v1.trace import Trace
+
+CLAUDE_HOME = "/tmp/vf-claude-code-{version}"
+CLAUDE_BIN = f"{CLAUDE_HOME}/.local/bin/claude"
+CLAUDE_CONFIG_DIR = ".vf-claude"
+INSTALL = """
+set -e
+command -v curl >/dev/null || (apt-get update -qq && apt-get install -y -qq curl ca-certificates >/dev/null)
+curl -fsSL https://claude.ai/install.sh | HOME={home} bash -s {version}
+"""
+
+
+class ClaudeCodeHarnessConfig(HarnessConfig):
+    version: str = Field(default="2.1.207", pattern=r"^[A-Za-z0-9._+-]+$")
+    """Claude Code release to install; pinned for reproducibility."""
+
+
+class ClaudeCodeHarness(Harness[ClaudeCodeHarnessConfig]):
+    APPENDS_SYSTEM_PROMPT = True
+    SUPPORTS_MCP = True
+    # images would require streaming inputs
+    SUPPORTS_MESSAGE_PROMPT = False
+
+    async def setup(self, runtime: Runtime) -> None:
+        home = CLAUDE_HOME.format(version=self.config.version)
+        binary = CLAUDE_BIN.format(version=self.config.version)
+        script = INSTALL.format(version=self.config.version, home=home)
+        # Cache the pinned binary across local rollouts; Linux has flock, macOS has lockf.
+        install = shlex.quote(f"[ -x {binary} ] || ({script})")
+        guarded = (
+            f"mkdir -p {home} && "
+            f'"$(command -v flock || command -v lockf)" {home}/install.lock '
+            f"bash -o pipefail -c {install}"
+        )
+        result = await runtime.run(["sh", "-c", guarded], self.config.resolved_env)
+        if result.exit_code != 0:
+            detail = (result.stderr or result.stdout).strip()[-500:]
+            raise RuntimeError(f"Claude Code install failed: {detail}")
+
+    async def launch(
+        self,
+        ctx: ModelContext,
+        trace: Trace,
+        runtime: Runtime,
+        endpoint: str,
+        secret: str,
+        mcp_urls: dict[str, str],
+    ) -> ProgramResult:
+        system_prompt, instruction = self.resolve_prompt(trace.task.data)
+        if ctx.client.base_url == "https://api.pinference.ai/api/v1":
+            # remove the /v1 from pinference
+            ctx.client.base_url = ctx.client.base_url.removesuffix("/v1")
+        env = {
+            **self.config.resolved_env,
+            # Claude appends /v1/messages; give it the interception root, not the model endpoint.
+            "ANTHROPIC_BASE_URL": endpoint.removesuffix("/v1"),
+            "ANTHROPIC_API_KEY": secret,
+            "CLAUDE_CONFIG_DIR": CLAUDE_CONFIG_DIR,
+            "DISABLE_AUTOUPDATER": "1",
+            "IS_SANDBOX": "1",
+        }
+        argv = [
+            CLAUDE_BIN.format(version=self.config.version),
+            "--print",
+            "--bare",
+            "--dangerously-skip-permissions",
+            "--no-session-persistence",
+            "--model",
+            ctx.model,
+        ]
+        if system_prompt:
+            argv += ["--append-system-prompt", system_prompt]
+        argv += [
+            arg
+            for tool in self.config.disabled_tools or []
+            for arg in ("--disallowedTools", tool)
+        ]
+        mcp = {
+            "mcpServers": {
+                name: {"type": "http", "url": url} for name, url in mcp_urls.items()
+            }
+        }
+        mcp_path = f"{CLAUDE_CONFIG_DIR}/mcp.json"
+        await runtime.write(mcp_path, json.dumps(mcp).encode())
+        argv += [
+            "--mcp-config",
+            mcp_path,
+            "--strict-mcp-config",
+            "--",
+            instruction or "",
+        ]
+        return await runtime.run_program(argv, env)

{verifiers-0.2.1.dev48 → verifiers-0.2.1.dev50}/verifiers/v1/tasksets/harbor/taskset.py

@@ -48,8 +48,14 @@ class HarborConfig(TasksetConfig):
     """Optional Harbor `--registry-url` selector for a raw registry.json URL."""
     tasks: list[str] | None = None
     """Optional subset of task names to load (None = all)."""
+    ignore_timeouts: bool = True
+    """Drop each task's declared agent and verifier timeouts so rollouts run
+    unbounded (unless run-level `--timeout.*` limits are set). Task timeouts are
+    authored against Harbor's runtime and confound model capability with inference
+    speed; set False to apply them anyway."""
     timeout_multiplier: float = Field(1.0, gt=0)
-    """Scale each task's agent and verifier timeouts."""
+    """Scale each task's agent and verifier timeouts. Only applies with
+    `ignore_timeouts=False`."""
     resource_multiplier: float = Field(1.0, gt=0)
     """Scale each task's CPU, memory, and disk requests. GPU requests are unchanged."""
     require_image: bool = False
@@ -273,8 +279,11 @@ def parse_task(task_dir: Path, idx: int, harbor_config: HarborConfig) -> HarborD
     # Older registry entries stored one author in [metadata].
     if not authors and meta.get("author_name"):
         authors = [Author(name=meta["author_name"], email=meta.get("author_email"))]
-    harness_timeout = config.get("agent", {}).get("timeout_sec")
-    scoring_timeout = config.get("verifier", {}).get("timeout_sec")
+    if harbor_config.ignore_timeouts:
+        harness_timeout = scoring_timeout = None
+    else:
+        harness_timeout = config.get("agent", {}).get("timeout_sec")
+        scoring_timeout = config.get("verifier", {}).get("timeout_sec")
     return HarborData(
         idx=idx,
         name=task.get("name") or task_dir.name,