verifiers 0.2.2.dev88__tar.gz → 0.2.2.dev90__tar.gz

{verifiers-0.2.2.dev88 → verifiers-0.2.2.dev90}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.2.dev88
+Version: 0.2.2.dev90
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers
@@ -35,7 +35,7 @@ Requires-Dist: numpy>=2.1.0
 Requires-Dist: openai-agents>=0.8.2
 Requires-Dist: openai>=2.9.0
 Requires-Dist: prime-pydantic-config[toml]>=0.4.2
-Requires-Dist: prime-sandboxes>=0.2.33
+Requires-Dist: prime-sandboxes>=0.2.35
 Requires-Dist: prime-tunnel>=0.1.8
 Requires-Dist: pydantic>=2.12.3
 Requires-Dist: pyzmq>=27.1.0

{verifiers-0.2.2.dev88 → verifiers-0.2.2.dev90}/pyproject.toml

@@ -35,7 +35,7 @@ dependencies = [
     "openai>=2.9.0",
     "openai-agents>=0.8.2",
     "prime-tunnel>=0.1.8",
-    "prime-sandboxes>=0.2.33",
+    "prime-sandboxes>=0.2.35",
     "pydantic>=2.12.3",
     "requests",
     "rich>=11.0.0",

{verifiers-0.2.2.dev88 → verifiers-0.2.2.dev90}/tests/v1/test_e2e.py

@@ -7,11 +7,11 @@ are local-only (their marks are excluded in CI)."""
 
 import pytest
 
-_m = pytest.mark
+mark = pytest.mark
 
 
-def _pair(a: str, b: str, id: str, *extra_marks):
-    marks = [getattr(_m, a.replace("-", "_")), getattr(_m, b.replace("-", "_"))]
+def pair(a: str, b: str, id: str, *extra_marks):
+    marks = [getattr(mark, a.replace("-", "_")), getattr(mark, b.replace("-", "_"))]
     return pytest.param(a, b, marks=[*marks, *extra_marks], id=id)
 
 
@@ -20,79 +20,83 @@ def _pair(a: str, b: str, id: str, *extra_marks):
 # provider. codex/claude-code are excluded here (unreliable on a no-op echo chat
 # task) — test_agentic covers them.
 CHAT_PLACEMENTS = [
-    _pair("null", "subprocess", "null-harness-in-subprocess"),
-    _pair("bash", "docker", "bash-harness-in-docker"),
-    _pair("rlm", "docker", "rlm-harness-in-docker"),
-    _pair("kimi-code", "docker", "kimi-code-harness-in-docker"),
-    _pair("bash", "prime", "bash-harness-in-prime"),
-    _pair("bash", "modal", "bash-harness-in-modal"),
+    pair("null", "subprocess", "null-harness-in-subprocess"),
+    pair("bash", "docker", "bash-harness-in-docker"),
+    pair("rlm", "docker", "rlm-harness-in-docker"),
+    pair("kimi-code", "docker", "kimi-code-harness-in-docker"),
+    pair("bash", "prime", "bash-harness-in-prime"),
+    pair("bash", "modal", "bash-harness-in-modal"),
 ]
 
 # harness x harness runtime for the shell task: every coding agent once (null is a chat
 # loop with no shell), both local runtimes hit (subprocess only carries bash), one
 # remote row per provider.
 AGENTIC_PLACEMENTS = [
-    _pair("bash", "subprocess", "bash-harness-in-subprocess"),
-    _pair("rlm", "docker", "rlm-harness-in-docker"),
-    _pair("kimi-code", "docker", "kimi-code-harness-in-docker"),
-    _pair("codex", "docker", "codex-harness-in-docker"),
-    _pair("claude-code", "docker", "claude-code-harness-in-docker"),
-    _pair("hermes-agent", "docker", "hermes-agent-harness-in-docker"),
-    _pair("bash", "prime", "bash-harness-in-prime"),
-    _pair("bash", "modal", "bash-harness-in-modal"),
+    pair("bash", "subprocess", "bash-harness-in-subprocess"),
+    pair("rlm", "docker", "rlm-harness-in-docker"),
+    pair("kimi-code", "docker", "kimi-code-harness-in-docker"),
+    pair("codex", "docker", "codex-harness-in-docker"),
+    pair("claude-code", "docker", "claude-code-harness-in-docker"),
+    pair("hermes-agent", "docker", "hermes-agent-harness-in-docker"),
+    pair("bash", "prime", "bash-harness-in-prime"),
+    pair("bash", "modal", "bash-harness-in-modal"),
 ]
 
 # The scripted user runs in the eval process itself (no placement axis); the harness
 # runtime is the exchange's only axis.
 USER_RUNTIMES = [
-    pytest.param("subprocess", marks=[_m.subprocess], id="harness-in-subprocess"),
-    pytest.param("docker", marks=[_m.docker], id="harness-in-docker"),
-    pytest.param("prime", marks=[_m.prime], id="harness-in-prime"),
-    pytest.param("modal", marks=[_m.modal], id="harness-in-modal"),
+    pytest.param("subprocess", marks=[mark.subprocess], id="harness-in-subprocess"),
+    pytest.param("docker", marks=[mark.docker], id="harness-in-docker"),
+    pytest.param("prime", marks=[mark.prime], id="harness-in-prime"),
+    pytest.param("modal", marks=[mark.modal], id="harness-in-modal"),
 ]
 
-# ACP-backed harnesses: each must preserve an exchange across process relaunches and
+# ACP-backed harnesses: each must preserve an exchange across interaction segments and
 # retain MCP access after resuming. Cover every harness in the local container runtime,
-# plus one remote placement for the sandbox/tunnel boundary.
+# plus remote placements for the sandbox/tunnel and native-process boundaries.
 ACP_RESUME_PLACEMENTS = [
-    _pair("hermes-agent", "docker", "hermes-agent-acp-in-docker"),
-    _pair("kimi-code", "docker", "kimi-code-acp-in-docker"),
-    _pair("pi", "docker", "pi-acp-in-docker"),
-    _pair("pool", "docker", "pool-acp-in-docker"),
-    _pair("openclaw", "docker", "openclaw-acp-in-docker"),
-    _pair("pool", "prime", "pool-acp-in-prime"),
+    pair("codex", "docker", "codex-acp-in-docker"),
+    pair("claude-code", "docker", "claude-code-acp-in-docker"),
+    pair("hermes-agent", "docker", "hermes-agent-acp-in-docker"),
+    pair("rlm", "docker", "rlm-acp-in-docker"),
+    pair("kimi-code", "docker", "kimi-code-acp-in-docker"),
+    pair("pi", "docker", "pi-acp-in-docker"),
+    pair("pool", "docker", "pool-acp-in-docker"),
+    pair("openclaw", "docker", "openclaw-acp-in-docker"),
+    pair("pool", "prime", "pool-acp-in-prime"),
+    pair("rlm", "prime", "rlm-acp-in-prime-vm"),
 ]
 
 # harness runtime x tool placement: every axis value once plus the two-container case
 # (harness and tool in separate docker boxes) and a prime-colocated row (a tool in its
 # OWN prime sandbox needs port exposure; colocated rides the harness's box).
 TOOL_PLACEMENTS = [
-    _pair("subprocess", "colocated", "harness-in-subprocess-with-tool-colocated"),
-    _pair("docker", "colocated", "harness-in-docker-with-tool-colocated"),
-    _pair("subprocess", "docker", "harness-in-subprocess-with-tool-in-docker"),
-    _pair("docker", "subprocess", "harness-in-docker-with-tool-in-subprocess"),
-    _pair("docker", "docker", "harness-in-docker-with-tool-in-docker"),
-    _pair("prime", "colocated", "harness-in-prime-with-tool-colocated"),
-    _pair("modal", "colocated", "harness-in-modal-with-tool-colocated"),
-    _pair("subprocess", "modal", "harness-in-subprocess-with-tool-in-modal"),
+    pair("subprocess", "colocated", "harness-in-subprocess-with-tool-colocated"),
+    pair("docker", "colocated", "harness-in-docker-with-tool-colocated"),
+    pair("subprocess", "docker", "harness-in-subprocess-with-tool-in-docker"),
+    pair("docker", "subprocess", "harness-in-docker-with-tool-in-subprocess"),
+    pair("docker", "docker", "harness-in-docker-with-tool-in-docker"),
+    pair("prime", "colocated", "harness-in-prime-with-tool-colocated"),
+    pair("modal", "colocated", "harness-in-modal-with-tool-colocated"),
+    pair("subprocess", "modal", "harness-in-subprocess-with-tool-in-modal"),
 ]
 
 # The state channel rides the same reachability as TOOL_PLACEMENTS; cover each axis
 # value once rather than re-running the whole list.
 TOOL_STATE_PLACEMENTS = [
-    _pair("subprocess", "colocated", "harness-in-subprocess-with-tool-colocated"),
-    _pair("docker", "subprocess", "harness-in-docker-with-tool-in-subprocess"),
-    _pair("subprocess", "docker", "harness-in-subprocess-with-tool-in-docker"),
-    _pair("modal", "colocated", "harness-in-modal-with-tool-colocated"),
+    pair("subprocess", "colocated", "harness-in-subprocess-with-tool-colocated"),
+    pair("docker", "subprocess", "harness-in-docker-with-tool-in-subprocess"),
+    pair("subprocess", "docker", "harness-in-subprocess-with-tool-in-docker"),
+    pair("modal", "colocated", "harness-in-modal-with-tool-colocated"),
 ]
 
 # Shared servers always run in their own runtime (colocation is per-rollout, shared is
 # eval-level): same-runtime pairs plus one cross-boundary row.
 SHARED_TOOL_PLACEMENTS = [
-    _pair("subprocess", "subprocess", "harness-in-subprocess-with-tool-in-subprocess"),
-    _pair("docker", "docker", "harness-in-docker-with-tool-in-docker"),
-    _pair("subprocess", "docker", "harness-in-subprocess-with-tool-in-docker"),
-    _pair("modal", "modal", "harness-in-modal-with-tool-in-modal"),
+    pair("subprocess", "subprocess", "harness-in-subprocess-with-tool-in-subprocess"),
+    pair("docker", "docker", "harness-in-docker-with-tool-in-docker"),
+    pair("subprocess", "docker", "harness-in-subprocess-with-tool-in-docker"),
+    pair("modal", "modal", "harness-in-modal-with-tool-in-modal"),
 ]
 
 
@@ -215,7 +219,10 @@ async def test_acp_resume_with_tool(run_v1, harness, harness_runtime, tmp_path):
     (trace,) = await run_v1(
         "echo-acp-resume-v1",
         harness=harness,
-        runtime={"type": harness_runtime},
+        runtime={
+            "type": harness_runtime,
+            **({"vm": True} if harness_runtime == "prime" else {}),
+        },
         output_dir=tmp_path,
         max_turns=8,
         max_tokens=8192,
@@ -231,6 +238,8 @@ async def test_acp_resume_with_tool(run_v1, harness, harness_runtime, tmp_path):
     assert segments[1]["terminated"] is False
     assert "tool" in segments[1]["roles"]
     assert segments[1]["tool_outputs"]
+    if harness == "rlm":
+        assert "turns_since_last_compaction" in trace.metrics
 
 
 @pytest.mark.e2e

{verifiers-0.2.2.dev88 → verifiers-0.2.2.dev90}/verifiers/v1/__init__.py

@@ -43,7 +43,7 @@ from verifiers.v1.errors import (
     TunnelError,
 )
 from verifiers.v1.graph import MessageNode
-from verifiers.v1.harness import Harness
+from verifiers.v1.harness import Harness, HarnessSession
 from verifiers.v1.judge import Judge, JudgeResponse, JudgeView
 from verifiers.v1.judges import (
     Criterion,
@@ -64,6 +64,7 @@ from verifiers.v1.runtimes import (
     Runtime,
     RuntimeConfig,
     RuntimeInfo,
+    RuntimeProcess,
     SubprocessConfig,
 )
 from verifiers.v1.state import State, StateT
@@ -241,10 +242,12 @@ __all__ = [  # noqa: RUF022 - grouped by public API area
     "TasksetConfig",
     "BaseConfig",
     "Harness",
+    "HarnessSession",
     "HarnessConfig",
     "ACP",
     "ModelContext",
     "Runtime",
+    "RuntimeProcess",
     "RuntimeConfig",
     "RuntimeInfo",
     "ProgramResult",

verifiers-0.2.2.dev90/verifiers/v1/acp/__init__.py

@@ -0,0 +1,312 @@
+"""Public Agent Client Protocol support for harness programs."""
+
+import asyncio
+import contextlib
+import json
+import secrets
+from collections.abc import AsyncIterator
+from pathlib import Path
+
+from verifiers.v1.clients import ModelContext
+from verifiers.v1.dialects.chat import message_to_wire
+from verifiers.v1.errors import HarnessError
+from verifiers.v1.harness import Harness, HarnessSession
+from verifiers.v1.runtimes import ProgramResult, Runtime, RuntimeProcess
+from verifiers.v1.task import TaskData
+from verifiers.v1.trace import Trace
+from verifiers.v1.types import Messages
+from verifiers.v1.utils.aio import run_shielded
+
+ACP_SOURCE = (Path(__file__).resolve().parent / "runner.py").read_text()
+MAX_PACKET_BYTES = 128 * 1024 * 1024
+
+__all__ = ["ACP"]
+
+
+class ACP:
+    """Run one-shot ACP agents or create rollout-scoped ACP sessions."""
+
+    async def setup(self, harness: Harness, runtime: Runtime) -> None:
+        await runtime.prepare_uv_script(
+            ACP_SOURCE, {**harness.config.resolved_env, "UV_FROZEN": "false"}
+        )
+
+    def session(
+        self,
+        harness: Harness,
+        ctx: ModelContext,
+        trace: Trace,
+        runtime: Runtime,
+        endpoint: str,
+        secret: str,
+        mcp_urls: dict[str, str],
+        data: TaskData,
+        *,
+        env: dict[str, str],
+        command: list[str],
+        prompt: str | Messages | None,
+        system_prompt: str | None = None,
+        session_meta: dict | None = None,
+    ) -> "ACPHarnessSession":
+        """Create a persistent ACP-backed handle owned by one rollout."""
+        return ACPHarnessSession(
+            harness,
+            ctx,
+            trace,
+            runtime,
+            endpoint,
+            secret,
+            mcp_urls,
+            data,
+            env=env,
+            command=command,
+            prompt=prompt,
+            system_prompt=system_prompt,
+            session_meta=session_meta,
+        )
+
+    async def run(
+        self,
+        runtime: Runtime,
+        env: dict[str, str],
+        command: list[str],
+        prompt: str | Messages | None,
+        *,
+        mcp_urls: dict[str, str] | None = None,
+        system_prompt: str | None = None,
+        session_path: str | None = None,
+        session_meta: dict | None = None,
+        allow_empty_tool_reply: bool = False,
+    ) -> ProgramResult:
+        """Run one ACP segment without retaining its process."""
+        return await self._run(
+            runtime,
+            env,
+            command,
+            prompt,
+            mcp_urls=mcp_urls,
+            system_prompt=system_prompt,
+            session_path=session_path,
+            session_meta=session_meta,
+            allow_empty_tool_reply=allow_empty_tool_reply,
+        )
+
+    async def _run(
+        self,
+        runtime: Runtime,
+        env: dict[str, str],
+        command: list[str],
+        prompt: str | Messages | None,
+        *,
+        mcp_urls: dict[str, str] | None = None,
+        system_prompt: str | None = None,
+        session_path: str | None = None,
+        session_meta: dict | None = None,
+        allow_empty_tool_reply: bool = False,
+    ) -> ProgramResult:
+        if prompt is None:
+            raise ValueError("ACP requires a prompt")
+        messages = (
+            [{"role": "user", "content": prompt}]
+            if isinstance(prompt, str)
+            else [message_to_wire(message) for message in prompt]
+        )
+        config = {
+            "command": command,
+            "messages": messages,
+            "mcp_urls": mcp_urls or {},
+            "system_prompt": system_prompt or "",
+            "session_path": session_path,
+            "session_meta": session_meta or {},
+            "allow_empty_tool_reply": allow_empty_tool_reply,
+        }
+        program = await runtime.prepare_uv_script(
+            ACP_SOURCE, {**env, "UV_FROZEN": "false"}
+        )
+        directory = f".vf-acp-{secrets.token_hex(8)}"
+        created = await runtime.run(["mkdir", "-m", "700", directory], {})
+        if created.exit_code != 0:
+            raise RuntimeError(f"ACP config directory failed: {created.stderr.strip()}")
+        path = f"{directory}/config.json"
+        try:
+            await runtime.write(path, json.dumps(config).encode())
+            return await runtime.run_program([*program, "once", path], env)
+        finally:
+            await run_shielded(runtime.run(["rm", "-rf", directory], {}))
+
+
+def _packet(value: dict) -> bytes:
+    data = json.dumps(value, ensure_ascii=False).encode()
+    if len(data) > MAX_PACKET_BYTES:
+        raise ValueError(f"ACP session packet is too large: {len(data)} bytes")
+    return len(data).to_bytes(8, "big") + data
+
+
+class _PacketReader:
+    def __init__(self, source: AsyncIterator[bytes]) -> None:
+        self._source = source.__aiter__()
+        self._buffer = bytearray()
+
+    async def _readexactly(self, size: int) -> bytes:
+        while len(self._buffer) < size:
+            try:
+                self._buffer.extend(await anext(self._source))
+            except StopAsyncIteration as e:
+                raise EOFError("ACP process closed its stdout") from e
+        data = bytes(self._buffer[:size])
+        del self._buffer[:size]
+        return data
+
+    async def read(self) -> dict:
+        size = int.from_bytes(await self._readexactly(8), "big")
+        if size > MAX_PACKET_BYTES:
+            raise ValueError(f"ACP session packet is too large: {size} bytes")
+        return json.loads((await self._readexactly(size)).decode())
+
+
+class ACPHarnessSession(HarnessSession):
+    """A live ACP process, connection, and native session for one rollout."""
+
+    def __init__(
+        self,
+        harness: Harness,
+        ctx: ModelContext,
+        trace: Trace,
+        runtime: Runtime,
+        endpoint: str,
+        secret: str,
+        mcp_urls: dict[str, str],
+        data: TaskData,
+        env: dict[str, str],
+        command: list[str],
+        prompt: str | Messages | None,
+        system_prompt: str | None,
+        session_meta: dict | None,
+    ) -> None:
+        super().__init__(harness, ctx, trace, runtime, endpoint, secret, mcp_urls, data)
+        self.env = env
+        self.command = command
+        self.prompt = prompt
+        self.system_prompt = system_prompt
+        self.session_meta = session_meta or {}
+        self._process: RuntimeProcess | None = None
+        self._reader: _PacketReader | None = None
+        self._stderr_tail = bytearray()
+        self._stderr_task: asyncio.Task[None] | None = None
+        self._lock = asyncio.Lock()
+
+    async def _start(self) -> None:
+        self._stderr_tail.clear()
+        program = await self.runtime.prepare_uv_script(
+            ACP_SOURCE, {**self.env, "UV_FROZEN": "false"}
+        )
+        process = await self.runtime.open_process([*program, "stream"], self.env)
+        self._process = process
+        self._reader = _PacketReader(process.stdout)
+        self._stderr_task = asyncio.create_task(self._drain_stderr(process.stderr))
+
+    async def _drain_stderr(self, stream: AsyncIterator[bytes]) -> None:
+        async for chunk in stream:
+            self._stderr_tail.extend(chunk)
+            if len(self._stderr_tail) > 4000:
+                del self._stderr_tail[:-4000]
+
+    def _stderr(self) -> str:
+        return self._stderr_tail.decode(errors="replace").strip()
+
+    async def _run(self, messages: Messages | None) -> ProgramResult:
+        prompt = self.prompt if messages is None else messages
+        if prompt is None:
+            raise ValueError("ACP requires a prompt")
+        wire_messages = (
+            [{"role": "user", "content": prompt}]
+            if isinstance(prompt, str)
+            else [message_to_wire(message) for message in prompt]
+        )
+        config = {
+            "command": self.command,
+            "messages": wire_messages,
+            "mcp_urls": self.mcp_urls,
+            "system_prompt": self.system_prompt or "",
+            "session_path": None,
+            "session_meta": self.session_meta,
+        }
+        async with self._lock:
+            if self._closed:
+                raise HarnessError(
+                    f"harness {self.harness.config.id!r} session is already closed"
+                )
+            if self._process is None:
+                await self._start()
+            assert self._process is not None
+            assert self._reader is not None
+            try:
+                await self._process.write(
+                    _packet({"operation": "prompt", "config": config})
+                )
+                response = await self._reader.read()
+            except BaseException:
+                await run_shielded(self._stop(graceful=False))
+                raise
+        if not response.get("ok"):
+            detail = response.get("error") or "ACP session request failed"
+            if stderr := self._stderr():
+                detail = f"{detail}\n\nACP process stderr:\n{stderr}"
+            raise RuntimeError(detail)
+        return ProgramResult(exit_code=0, stdout=response.get("reply", ""), stderr="")
+
+    async def _stop(self, *, graceful: bool) -> None:
+        process, self._process = self._process, None
+        reader, self._reader = self._reader, None
+        stderr_task, self._stderr_task = self._stderr_task, None
+        if process is None:
+            return
+        failure: BaseException | None = None
+        try:
+            if graceful and reader is not None:
+                try:
+                    await process.write(_packet({"operation": "shutdown"}))
+                    response = await asyncio.wait_for(reader.read(), timeout=10)
+                    if not response.get("ok"):
+                        raise RuntimeError(
+                            response.get("error") or "ACP session shutdown failed"
+                        )
+                except BaseException as error:  # noqa: BLE001 - finish teardown if cancelled
+                    failure = error
+            try:
+                await asyncio.wait_for(process.wait(), timeout=10 if graceful else 0.1)
+            except BaseException:  # noqa: BLE001 - cancellation still requires termination
+                with contextlib.suppress(Exception):
+                    await asyncio.wait_for(process.terminate(), timeout=5)
+                try:
+                    await asyncio.wait_for(process.wait(), timeout=5)
+                except BaseException:  # noqa: BLE001 - cancellation still requires a kill
+                    with contextlib.suppress(Exception):
+                        await asyncio.wait_for(process.kill(), timeout=5)
+                    with contextlib.suppress(BaseException):
+                        await asyncio.wait_for(process.wait(), timeout=5)
+        finally:
+            if stderr_task is not None:
+                if not stderr_task.done():
+                    stderr_task.cancel()
+                with contextlib.suppress(BaseException):
+                    await stderr_task
+        if failure is not None:
+            detail = str(failure)
+            if stderr := self._stderr():
+                detail = f"{detail}\n\nACP process stderr:\n{stderr}"
+            raise RuntimeError(detail) from failure
+
+    async def close(self) -> None:
+        if self._closed:
+            return
+        # Publish closure before waiting for the process lock. A turn that
+        # already passed HarnessSession.turn()'s fast check rechecks under the
+        # same lock in _run(), so it cannot restart after teardown.
+        await super().close()
+
+        async def close_process() -> None:
+            async with self._lock:
+                await self._stop(graceful=True)
+
+        await run_shielded(close_process())