verifiers 0.2.2.dev89__tar.gz → 0.2.2.dev90__tar.gz

{verifiers-0.2.2.dev89 → verifiers-0.2.2.dev90}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.2.dev89
+Version: 0.2.2.dev90
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.2.2.dev89 → verifiers-0.2.2.dev90}/tests/v1/test_e2e.py

@@ -7,11 +7,11 @@ are local-only (their marks are excluded in CI)."""
 
 import pytest
 
-_m = pytest.mark
+mark = pytest.mark
 
 
-def _pair(a: str, b: str, id: str, *extra_marks):
-    marks = [getattr(_m, a.replace("-", "_")), getattr(_m, b.replace("-", "_"))]
+def pair(a: str, b: str, id: str, *extra_marks):
+    marks = [getattr(mark, a.replace("-", "_")), getattr(mark, b.replace("-", "_"))]
     return pytest.param(a, b, marks=[*marks, *extra_marks], id=id)
 
 
@@ -20,81 +20,83 @@ def _pair(a: str, b: str, id: str, *extra_marks):
 # provider. codex/claude-code are excluded here (unreliable on a no-op echo chat
 # task) — test_agentic covers them.
 CHAT_PLACEMENTS = [
-    _pair("null", "subprocess", "null-harness-in-subprocess"),
-    _pair("bash", "docker", "bash-harness-in-docker"),
-    _pair("rlm", "docker", "rlm-harness-in-docker"),
-    _pair("kimi-code", "docker", "kimi-code-harness-in-docker"),
-    _pair("bash", "prime", "bash-harness-in-prime"),
-    _pair("bash", "modal", "bash-harness-in-modal"),
+    pair("null", "subprocess", "null-harness-in-subprocess"),
+    pair("bash", "docker", "bash-harness-in-docker"),
+    pair("rlm", "docker", "rlm-harness-in-docker"),
+    pair("kimi-code", "docker", "kimi-code-harness-in-docker"),
+    pair("bash", "prime", "bash-harness-in-prime"),
+    pair("bash", "modal", "bash-harness-in-modal"),
 ]
 
 # harness x harness runtime for the shell task: every coding agent once (null is a chat
 # loop with no shell), both local runtimes hit (subprocess only carries bash), one
 # remote row per provider.
 AGENTIC_PLACEMENTS = [
-    _pair("bash", "subprocess", "bash-harness-in-subprocess"),
-    _pair("rlm", "docker", "rlm-harness-in-docker"),
-    _pair("kimi-code", "docker", "kimi-code-harness-in-docker"),
-    _pair("codex", "docker", "codex-harness-in-docker"),
-    _pair("claude-code", "docker", "claude-code-harness-in-docker"),
-    _pair("hermes-agent", "docker", "hermes-agent-harness-in-docker"),
-    _pair("bash", "prime", "bash-harness-in-prime"),
-    _pair("bash", "modal", "bash-harness-in-modal"),
+    pair("bash", "subprocess", "bash-harness-in-subprocess"),
+    pair("rlm", "docker", "rlm-harness-in-docker"),
+    pair("kimi-code", "docker", "kimi-code-harness-in-docker"),
+    pair("codex", "docker", "codex-harness-in-docker"),
+    pair("claude-code", "docker", "claude-code-harness-in-docker"),
+    pair("hermes-agent", "docker", "hermes-agent-harness-in-docker"),
+    pair("bash", "prime", "bash-harness-in-prime"),
+    pair("bash", "modal", "bash-harness-in-modal"),
 ]
 
 # The scripted user runs in the eval process itself (no placement axis); the harness
 # runtime is the exchange's only axis.
 USER_RUNTIMES = [
-    pytest.param("subprocess", marks=[_m.subprocess], id="harness-in-subprocess"),
-    pytest.param("docker", marks=[_m.docker], id="harness-in-docker"),
-    pytest.param("prime", marks=[_m.prime], id="harness-in-prime"),
-    pytest.param("modal", marks=[_m.modal], id="harness-in-modal"),
+    pytest.param("subprocess", marks=[mark.subprocess], id="harness-in-subprocess"),
+    pytest.param("docker", marks=[mark.docker], id="harness-in-docker"),
+    pytest.param("prime", marks=[mark.prime], id="harness-in-prime"),
+    pytest.param("modal", marks=[mark.modal], id="harness-in-modal"),
 ]
 
 # ACP-backed harnesses: each must preserve an exchange across interaction segments and
 # retain MCP access after resuming. Cover every harness in the local container runtime,
 # plus remote placements for the sandbox/tunnel and native-process boundaries.
 ACP_RESUME_PLACEMENTS = [
-    _pair("hermes-agent", "docker", "hermes-agent-acp-in-docker"),
-    _pair("rlm", "docker", "rlm-acp-in-docker"),
-    _pair("kimi-code", "docker", "kimi-code-acp-in-docker"),
-    _pair("pi", "docker", "pi-acp-in-docker"),
-    _pair("pool", "docker", "pool-acp-in-docker"),
-    _pair("openclaw", "docker", "openclaw-acp-in-docker"),
-    _pair("pool", "prime", "pool-acp-in-prime"),
-    _pair("rlm", "prime", "rlm-acp-in-prime-vm"),
+    pair("codex", "docker", "codex-acp-in-docker"),
+    pair("claude-code", "docker", "claude-code-acp-in-docker"),
+    pair("hermes-agent", "docker", "hermes-agent-acp-in-docker"),
+    pair("rlm", "docker", "rlm-acp-in-docker"),
+    pair("kimi-code", "docker", "kimi-code-acp-in-docker"),
+    pair("pi", "docker", "pi-acp-in-docker"),
+    pair("pool", "docker", "pool-acp-in-docker"),
+    pair("openclaw", "docker", "openclaw-acp-in-docker"),
+    pair("pool", "prime", "pool-acp-in-prime"),
+    pair("rlm", "prime", "rlm-acp-in-prime-vm"),
 ]
 
 # harness runtime x tool placement: every axis value once plus the two-container case
 # (harness and tool in separate docker boxes) and a prime-colocated row (a tool in its
 # OWN prime sandbox needs port exposure; colocated rides the harness's box).
 TOOL_PLACEMENTS = [
-    _pair("subprocess", "colocated", "harness-in-subprocess-with-tool-colocated"),
-    _pair("docker", "colocated", "harness-in-docker-with-tool-colocated"),
-    _pair("subprocess", "docker", "harness-in-subprocess-with-tool-in-docker"),
-    _pair("docker", "subprocess", "harness-in-docker-with-tool-in-subprocess"),
-    _pair("docker", "docker", "harness-in-docker-with-tool-in-docker"),
-    _pair("prime", "colocated", "harness-in-prime-with-tool-colocated"),
-    _pair("modal", "colocated", "harness-in-modal-with-tool-colocated"),
-    _pair("subprocess", "modal", "harness-in-subprocess-with-tool-in-modal"),
+    pair("subprocess", "colocated", "harness-in-subprocess-with-tool-colocated"),
+    pair("docker", "colocated", "harness-in-docker-with-tool-colocated"),
+    pair("subprocess", "docker", "harness-in-subprocess-with-tool-in-docker"),
+    pair("docker", "subprocess", "harness-in-docker-with-tool-in-subprocess"),
+    pair("docker", "docker", "harness-in-docker-with-tool-in-docker"),
+    pair("prime", "colocated", "harness-in-prime-with-tool-colocated"),
+    pair("modal", "colocated", "harness-in-modal-with-tool-colocated"),
+    pair("subprocess", "modal", "harness-in-subprocess-with-tool-in-modal"),
 ]
 
 # The state channel rides the same reachability as TOOL_PLACEMENTS; cover each axis
 # value once rather than re-running the whole list.
 TOOL_STATE_PLACEMENTS = [
-    _pair("subprocess", "colocated", "harness-in-subprocess-with-tool-colocated"),
-    _pair("docker", "subprocess", "harness-in-docker-with-tool-in-subprocess"),
-    _pair("subprocess", "docker", "harness-in-subprocess-with-tool-in-docker"),
-    _pair("modal", "colocated", "harness-in-modal-with-tool-colocated"),
+    pair("subprocess", "colocated", "harness-in-subprocess-with-tool-colocated"),
+    pair("docker", "subprocess", "harness-in-docker-with-tool-in-subprocess"),
+    pair("subprocess", "docker", "harness-in-subprocess-with-tool-in-docker"),
+    pair("modal", "colocated", "harness-in-modal-with-tool-colocated"),
 ]
 
 # Shared servers always run in their own runtime (colocation is per-rollout, shared is
 # eval-level): same-runtime pairs plus one cross-boundary row.
 SHARED_TOOL_PLACEMENTS = [
-    _pair("subprocess", "subprocess", "harness-in-subprocess-with-tool-in-subprocess"),
-    _pair("docker", "docker", "harness-in-docker-with-tool-in-docker"),
-    _pair("subprocess", "docker", "harness-in-subprocess-with-tool-in-docker"),
-    _pair("modal", "modal", "harness-in-modal-with-tool-in-modal"),
+    pair("subprocess", "subprocess", "harness-in-subprocess-with-tool-in-subprocess"),
+    pair("docker", "docker", "harness-in-docker-with-tool-in-docker"),
+    pair("subprocess", "docker", "harness-in-subprocess-with-tool-in-docker"),
+    pair("modal", "modal", "harness-in-modal-with-tool-in-modal"),
 ]
 
 

{verifiers-0.2.2.dev89 → verifiers-0.2.2.dev90}/verifiers/v1/acp/__init__.py

@@ -46,6 +46,7 @@ class ACP:
         command: list[str],
         prompt: str | Messages | None,
         system_prompt: str | None = None,
+        session_meta: dict | None = None,
     ) -> "ACPHarnessSession":
         """Create a persistent ACP-backed handle owned by one rollout."""
         return ACPHarnessSession(
@@ -61,6 +62,7 @@ class ACP:
             command=command,
             prompt=prompt,
             system_prompt=system_prompt,
+            session_meta=session_meta,
         )
 
     async def run(
@@ -73,6 +75,7 @@ class ACP:
         mcp_urls: dict[str, str] | None = None,
         system_prompt: str | None = None,
         session_path: str | None = None,
+        session_meta: dict | None = None,
         allow_empty_tool_reply: bool = False,
     ) -> ProgramResult:
         """Run one ACP segment without retaining its process."""
@@ -84,6 +87,7 @@ class ACP:
             mcp_urls=mcp_urls,
             system_prompt=system_prompt,
             session_path=session_path,
+            session_meta=session_meta,
             allow_empty_tool_reply=allow_empty_tool_reply,
         )
 
@@ -97,6 +101,7 @@ class ACP:
         mcp_urls: dict[str, str] | None = None,
         system_prompt: str | None = None,
         session_path: str | None = None,
+        session_meta: dict | None = None,
         allow_empty_tool_reply: bool = False,
     ) -> ProgramResult:
         if prompt is None:
@@ -112,6 +117,7 @@ class ACP:
             "mcp_urls": mcp_urls or {},
             "system_prompt": system_prompt or "",
             "session_path": session_path,
+            "session_meta": session_meta or {},
             "allow_empty_tool_reply": allow_empty_tool_reply,
         }
         program = await runtime.prepare_uv_script(
@@ -175,12 +181,14 @@ class ACPHarnessSession(HarnessSession):
         command: list[str],
         prompt: str | Messages | None,
         system_prompt: str | None,
+        session_meta: dict | None,
     ) -> None:
         super().__init__(harness, ctx, trace, runtime, endpoint, secret, mcp_urls, data)
         self.env = env
         self.command = command
         self.prompt = prompt
         self.system_prompt = system_prompt
+        self.session_meta = session_meta or {}
         self._process: RuntimeProcess | None = None
         self._reader: _PacketReader | None = None
         self._stderr_tail = bytearray()
@@ -221,6 +229,7 @@ class ACPHarnessSession(HarnessSession):
             "mcp_urls": self.mcp_urls,
             "system_prompt": self.system_prompt or "",
             "session_path": None,
+            "session_meta": self.session_meta,
         }
         async with self._lock:
             if self._closed:

{verifiers-0.2.2.dev89 → verifiers-0.2.2.dev90}/verifiers/v1/acp/runner.py

@@ -196,28 +196,38 @@ async def run_once(config: dict) -> str:
         *command[1:],
         env=os.environ.copy(),
         transport_kwargs={"stderr": None},
-    ) as (connection, _process):
+    ) as agent_process:
+        connection = agent_process[0]
         initialized = await connection.initialize(
             protocol_version=PROTOCOL_VERSION,
             client_capabilities=ClientCapabilities(),
         )
         capabilities = initialized.agent_capabilities
         session_path = Path(config["session_path"]) if config["session_path"] else None
+        session_meta = config["session_meta"]
         is_new = session_path is None or not session_path.exists()
         servers = mcp_servers(config)
         if is_new:
-            session = await connection.new_session(cwd=os.getcwd(), mcp_servers=servers)
+            session = await connection.new_session(
+                cwd=os.getcwd(), mcp_servers=servers, **session_meta
+            )
             session_id = session.session_id
         else:
             session_id = session_path.read_text().strip()
             session_capabilities = capabilities and capabilities.session_capabilities
             if session_capabilities and session_capabilities.resume is not None:
                 await connection.resume_session(
-                    cwd=os.getcwd(), session_id=session_id, mcp_servers=servers
+                    cwd=os.getcwd(),
+                    session_id=session_id,
+                    mcp_servers=servers,
+                    **session_meta,
                 )
             elif capabilities and capabilities.load_session:
                 await connection.load_session(
-                    cwd=os.getcwd(), session_id=session_id, mcp_servers=servers
+                    cwd=os.getcwd(),
+                    session_id=session_id,
+                    mcp_servers=servers,
+                    **session_meta,
                 )
             else:
                 raise RuntimeError("ACP agent does not support resuming sessions")
@@ -251,12 +261,13 @@ class LiveACPSession:
         self.command: list[str] | None = None
         self.server_urls: dict[str, str] | None = None
         self.system_prompt: str | None = None
+        self.session_meta: dict | None = None
         self.is_new = True
 
     async def start(self, config: dict) -> None:
         command = config["command"]
         try:
-            self.connection, _process = await self.stack.enter_async_context(
+            agent_process = await self.stack.enter_async_context(
                 spawn_agent_process(
                     self.client,
                     command[0],
@@ -265,13 +276,16 @@ class LiveACPSession:
                     transport_kwargs={"stderr": None},
                 )
             )
+            self.connection = agent_process[0]
             initialized = await self.connection.initialize(
                 protocol_version=PROTOCOL_VERSION,
                 client_capabilities=ClientCapabilities(),
             )
             self.capabilities = initialized.agent_capabilities
             session = await self.connection.new_session(
-                cwd=os.getcwd(), mcp_servers=mcp_servers(config)
+                cwd=os.getcwd(),
+                mcp_servers=mcp_servers(config),
+                **config["session_meta"],
             )
         except BaseException:
             with suppress(BaseException):
@@ -282,6 +296,7 @@ class LiveACPSession:
         self.command = command
         self.server_urls = config["mcp_urls"]
         self.system_prompt = config["system_prompt"]
+        self.session_meta = config["session_meta"]
         self.is_new = True
 
     async def run(self, config: dict) -> str:
@@ -291,6 +306,7 @@ class LiveACPSession:
             config["command"] != self.command
             or config["mcp_urls"] != self.server_urls
             or config["system_prompt"] != self.system_prompt
+            or config["session_meta"] != self.session_meta
         ):
             raise RuntimeError("ACP session configuration changed")
         assert self.session_id is not None

verifiers-0.2.2.dev90/verifiers/v1/harnesses/claude_code/harness.py

@@ -0,0 +1,165 @@
+"""Run Claude Code through the Claude Agent SDK ACP adapter."""
+
+import shlex
+
+from verifiers.v1.acp import ACP
+from verifiers.v1.clients import ModelContext
+from verifiers.v1.configs.harness import HarnessConfig
+from verifiers.v1.harness import Harness, HarnessSession
+from verifiers.v1.harnesses.node import NODE_BIN_DIR, ensure_node
+from verifiers.v1.runtimes import ProgramResult, Runtime
+from verifiers.v1.task import TaskData
+from verifiers.v1.trace import Trace
+
+CLAUDE_ACP_DIR = "/var/tmp/vf-claude-agent-acp"
+PACKAGES_DIR = f"{CLAUDE_ACP_DIR}/packages"
+ACP_VERSION = "0.63.0"
+ACP_BIN = f"{PACKAGES_DIR}/node_modules/.bin/claude-agent-acp"
+ACP_COMMAND = [f"{NODE_BIN_DIR}/node", ACP_BIN]
+CLAUDE_CONFIG_ROOT = ".vf-claude"
+SKILLS_DIR = ".claude/skills"
+ACP_INSTALL = r"""
+set -e
+export PATH="/var/tmp/vf-node/bin:$PATH"
+if [ "$(cat /var/tmp/vf-claude-agent-acp/.version 2>/dev/null)" = "$VF_CLAUDE_ACP_VERSION" ] \
+    && [ -x /var/tmp/vf-claude-agent-acp/packages/node_modules/.bin/claude-agent-acp ]; then
+    exit 0
+fi
+npm install --prefix /var/tmp/vf-claude-agent-acp/packages --ignore-scripts --no-audit --no-fund \
+    --omit=dev \
+    "@agentclientprotocol/claude-agent-acp@$VF_CLAUDE_ACP_VERSION" >/dev/null
+printf %s "$VF_CLAUDE_ACP_VERSION" > /var/tmp/vf-claude-agent-acp/.version
+"""
+
+CLAUDE_ACP = ACP()
+
+
+class ClaudeCodeHarnessConfig(HarnessConfig):
+    pass
+
+
+class ClaudeCodeHarness(Harness[ClaudeCodeHarnessConfig]):
+    APPENDS_SYSTEM_PROMPT = True
+    SUPPORTS_MCP = True
+    SUPPORTS_RESUME = True
+    SUPPORTS_SKILLS = True
+
+    async def setup(self, runtime: Runtime) -> None:
+        await self.install_skills(runtime, SKILLS_DIR)
+        await ensure_node(runtime)
+        acp_guarded = (
+            f"mkdir -p {CLAUDE_ACP_DIR} && "
+            f'"$(command -v flock || command -v lockf)" {CLAUDE_ACP_DIR}/install.lock '
+            f"sh -c {shlex.quote(ACP_INSTALL)}"
+        )
+        acp_result = await runtime.run(
+            ["sh", "-c", acp_guarded],
+            {
+                **self.config.resolved_env,
+                "VF_CLAUDE_ACP_VERSION": ACP_VERSION,
+            },
+        )
+        if acp_result.exit_code != 0:
+            detail = (acp_result.stderr or acp_result.stdout).strip()[-500:]
+            raise RuntimeError(f"Claude Agent ACP install failed: {detail}")
+        await CLAUDE_ACP.setup(self, runtime)
+
+    async def session(
+        self,
+        ctx: ModelContext,
+        trace: Trace,
+        runtime: Runtime,
+        endpoint: str,
+        secret: str,
+        mcp_urls: dict[str, str],
+        data: TaskData,
+    ) -> HarnessSession:
+        if not runtime.supports_live_processes:
+            return await super().session(
+                ctx, trace, runtime, endpoint, secret, mcp_urls, data
+            )
+        system_prompt, prompt = self.resolve_prompt(data)
+        config_dir = self.config_dir(trace)
+        options: dict[str, object] = {
+            "strictMcpConfig": True,
+            "disallowedTools": self.config.disabled_tools or [],
+        }
+        session_meta: dict[str, object] = {"claudeCode": {"options": options}}
+        if system_prompt:
+            session_meta["systemPrompt"] = {"append": system_prompt}
+        env = {
+            **self.config.resolved_env,
+            "ANTHROPIC_BASE_URL": endpoint.removesuffix("/v1"),
+            "ANTHROPIC_API_KEY": secret,
+            "ANTHROPIC_MODEL": ctx.model,
+            "CLAUDE_CONFIG_DIR": config_dir,
+            "CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC": "1",
+            "DISABLE_AUTOUPDATER": "1",
+            "IS_SANDBOX": "1",
+        }
+        return CLAUDE_ACP.session(
+            self,
+            ctx,
+            trace,
+            runtime,
+            endpoint,
+            secret,
+            mcp_urls,
+            data,
+            env=env,
+            command=ACP_COMMAND,
+            prompt=prompt or "",
+            session_meta=session_meta,
+        )
+
+    async def launch(
+        self,
+        ctx: ModelContext,
+        trace: Trace,
+        runtime: Runtime,
+        endpoint: str,
+        secret: str,
+        mcp_urls: dict[str, str],
+        data: TaskData,
+    ) -> ProgramResult:
+        system_prompt, prompt = self.resolve_prompt(data)
+        config_dir = self.config_dir(trace)
+
+        options: dict[str, object] = {
+            "strictMcpConfig": True,
+            "disallowedTools": self.config.disabled_tools or [],
+        }
+        session_meta: dict[str, object] = {"claudeCode": {"options": options}}
+        if system_prompt:
+            session_meta["systemPrompt"] = {"append": system_prompt}
+        env = {
+            **self.config.resolved_env,
+            # Claude appends /v1/messages; give it the interception root, not the model endpoint.
+            "ANTHROPIC_BASE_URL": endpoint.removesuffix("/v1"),
+            "ANTHROPIC_API_KEY": secret,
+            "ANTHROPIC_MODEL": ctx.model,
+            "CLAUDE_CONFIG_DIR": config_dir,
+            "CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC": "1",
+            "DISABLE_AUTOUPDATER": "1",
+            "IS_SANDBOX": "1",
+        }
+        return await CLAUDE_ACP.run(
+            runtime,
+            env,
+            ACP_COMMAND,
+            prompt or "",
+            mcp_urls=mcp_urls,
+            session_path=f"{config_dir}/acp-session",
+            session_meta=session_meta,
+        )
+
+    async def cleanup(self, trace: Trace, runtime: Runtime) -> None:
+        result = await runtime.run(["rm", "-rf", self.config_dir(trace)], {})
+        if result.exit_code != 0:
+            raise RuntimeError(
+                f"failed to clean up Claude config: {result.stderr.strip()[-500:]}"
+            )
+
+    @staticmethod
+    def config_dir(trace: Trace) -> str:
+        return f"{CLAUDE_CONFIG_ROOT}/{trace.id}"