verifiers 0.3.1.dev22__tar.gz → 0.3.1.dev23__tar.gz

{verifiers-0.3.1.dev22 → verifiers-0.3.1.dev23}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.3.1.dev22
+Version: 0.3.1.dev23
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.3.1.dev22 → verifiers-0.3.1.dev23}/tests/v1/conftest.py

@@ -40,8 +40,9 @@ import pytest
 import verifiers.v1 as vf
 from verifiers.v1.cli.eval.runner import run_eval
 from verifiers.v1.configs.cli.eval import EvalConfig
+from verifiers.v1.configs.harness import HarnessConfig
 from verifiers.v1.trace import Trace
-from verifiers.v1.utils.loaders import load_environment
+from verifiers.v1.utils.loaders import harness_config_type, load_environment
 
 CI_MODEL = "openai/gpt-5.6-luna"
 
@@ -74,8 +75,9 @@ def tool_runtime(request) -> dict:
 # dependencies at rollout.
 # `compact` (an example harness) and `terminus-2` (drives the host tmux) are excluded from e2e.
 @pytest.fixture
-def harness(request) -> str:
-    return request.param
+def harness(request) -> HarnessConfig:
+    config = {"id": request.param} if isinstance(request.param, str) else request.param
+    return harness_config_type(config["id"]).model_validate(config)
 
 
 def pytest_configure(config) -> None:
@@ -119,7 +121,7 @@ def _eval_config(
     taskset: str,
     *,
     output_dir: Path,
-    harness: str | None = "null",
+    harness: str | HarnessConfig | None = "null",
     n: int = 1,
     num_tasks: int = 1,
     max_tokens: int = 2048,
@@ -143,7 +145,11 @@ def _eval_config(
     env_cfg = dict(env or {})
     _configure_prime_runtimes(taskset_cfg)
     if harness:
-        env_cfg.setdefault("agent", {})["harness"] = {"id": harness}
+        env_cfg.setdefault("agent", {})["harness"] = (
+            harness_config_type(harness)(id=harness)
+            if isinstance(harness, str)
+            else harness
+        )
     if runtime:
         runtime_cfg = dict(runtime)
         _configure_prime_runtimes(runtime_cfg)

{verifiers-0.3.1.dev22 → verifiers-0.3.1.dev23}/tests/v1/test_e2e.py

@@ -23,7 +23,12 @@ CHAT_PLACEMENTS = [
     pair("null", "subprocess", "null-harness-in-subprocess"),
     pair("bash", "docker", "bash-harness-in-docker"),
     pair("rlm", "docker", "rlm-harness-in-docker"),
-    pair("kimi-code", "docker", "kimi-code-harness-in-docker"),
+    pytest.param(
+        {"id": "kimi-code", "transport": "responses"},
+        "docker",
+        marks=[mark.kimi_code, mark.docker],
+        id="kimi-code-responses-harness-in-docker",
+    ),
     pair("bash", "prime", "bash-harness-in-prime"),
     pair("bash", "modal", "bash-harness-in-modal"),
 ]
@@ -34,7 +39,12 @@ CHAT_PLACEMENTS = [
 AGENTIC_PLACEMENTS = [
     pair("bash", "subprocess", "bash-harness-in-subprocess"),
     pair("rlm", "docker", "rlm-harness-in-docker"),
-    pair("kimi-code", "docker", "kimi-code-harness-in-docker"),
+    pytest.param(
+        {"id": "kimi-code", "transport": "responses"},
+        "docker",
+        marks=[mark.kimi_code, mark.docker],
+        id="kimi-code-responses-harness-in-docker",
+    ),
     pair("codex", "docker", "codex-harness-in-docker"),
     pair("claude-code", "docker", "claude-code-harness-in-docker"),
     pair("hermes-agent", "docker", "hermes-agent-harness-in-docker"),
@@ -59,8 +69,18 @@ ACP_RESUME_PLACEMENTS = [
     pair("claude-code", "docker", "claude-code-acp-in-docker"),
     pair("hermes-agent", "docker", "hermes-agent-acp-in-docker"),
     pair("rlm", "docker", "rlm-acp-in-docker"),
-    pair("kimi-code", "docker", "kimi-code-acp-in-docker"),
-    pair("pi", "docker", "pi-acp-in-docker"),
+    pytest.param(
+        {"id": "kimi-code", "transport": "responses"},
+        "docker",
+        marks=[mark.kimi_code, mark.docker],
+        id="kimi-code-responses-acp-in-docker",
+    ),
+    pytest.param(
+        {"id": "pi", "transport": "responses"},
+        "docker",
+        marks=[mark.pi, mark.docker],
+        id="pi-responses-acp-in-docker",
+    ),
     pair("pool", "docker", "pool-acp-in-docker"),
     pair("openclaw", "docker", "openclaw-acp-in-docker"),
     pair("pool", "prime", "pool-acp-in-prime"),
@@ -235,11 +255,14 @@ async def test_acp_resume_with_tool(run_v1, harness, harness_runtime, tmp_path):
     assert len(segments) == 2
     assert segments[0]["terminated"] is False
     assert segments[1]["terminated"] is False
+    # Kimi Code is broken upstream: its Responses adapter drops message `phase` on replay.
+    if harness.id != "kimi-code":
+        assert trace.num_branches == 1
     # Native MCP tools need not appear in the intercepted model request that
     # populates trace.tools; the ACP transcript is the source of truth for use.
     assert "tool" in segments[1]["roles"]
     assert segments[1]["tool_outputs"]
-    if harness == "rlm":
+    if harness.id == "rlm":
         assert "turns_since_last_compaction" in trace.metrics
 
 

{verifiers-0.3.1.dev22 → verifiers-0.3.1.dev23}/verifiers/v1/__init__.py

@@ -2,7 +2,7 @@ import logging as _logging
 
 from pydantic_config import BaseConfig
 
-from verifiers.v1.acp import ACP
+from verifiers.v1.acp import ACPConfig, ACPHarness
 from verifiers.v1.agent import Agent, Agents, Interaction, Segment, make_agent
 from verifiers.v1.clients import (
     BaseClientConfig,
@@ -249,7 +249,8 @@ __all__ = [  # noqa: RUF022 - grouped by public API area
     "Harness",
     "HarnessSession",
     "HarnessConfig",
-    "ACP",
+    "ACPConfig",
+    "ACPHarness",
     "ModelContext",
     "Runtime",
     "RuntimeProcess",

{verifiers-0.3.1.dev22 → verifiers-0.3.1.dev23}/verifiers/v1/acp/__init__.py

@@ -3,12 +3,14 @@
 import asyncio
 import contextlib
 import json
-import secrets
+from abc import abstractmethod
 from collections.abc import AsyncIterator
+from dataclasses import dataclass
 from pathlib import Path
+from typing import TypeAlias, TypeVar
 
 from verifiers.v1.clients import ModelContext
-from verifiers.v1.dialects.chat import message_to_wire
+from verifiers.v1.configs.harness import HarnessConfig
 from verifiers.v1.errors import HarnessError
 from verifiers.v1.harness import Harness, HarnessSession
 from verifiers.v1.runtimes import ProgramResult, Runtime, RuntimeProcess
@@ -20,20 +22,39 @@ from verifiers.v1.utils.aio import run_shielded
 ACP_SOURCE = (Path(__file__).resolve().parent / "runner.py").read_text()
 MAX_PACKET_BYTES = 128 * 1024 * 1024
 
-__all__ = ["ACP"]
+__all__ = ["ACPConfig", "ACPHarness"]
 
+ConfigT = TypeVar("ConfigT", bound=HarnessConfig)
+JsonValue: TypeAlias = (
+    str | int | float | bool | None | list["JsonValue"] | dict[str, "JsonValue"]
+)
+JsonObject: TypeAlias = dict[str, JsonValue]
 
-class ACP:
-    """Run one-shot ACP agents or create rollout-scoped ACP sessions."""
 
-    async def setup(self, harness: Harness, runtime: Runtime) -> None:
+@dataclass
+class ACPConfig:
+    """One harness's ACP process and initial prompt."""
+
+    env: dict[str, str]
+    command: list[str]
+    prompt: str | Messages | None
+    mcp_urls: dict[str, str] | None = None
+    system_prompt: str | None = None
+    session_meta: JsonObject | None = None
+    allow_empty_tool_reply: bool = False
+
+
+class ACPHarness(Harness[ConfigT]):
+    """Harness backed by one live ACP process and native session per rollout."""
+
+    async def setup(self, runtime: Runtime) -> None:
         await runtime.prepare_uv_script(
-            ACP_SOURCE, {**harness.config.resolved_env, "UV_FROZEN": "false"}
+            ACP_SOURCE, {**self.config.resolved_env, "UV_FROZEN": "false"}
         )
 
-    def session(
+    @abstractmethod
+    async def prepare_acp(
         self,
-        harness: Harness,
         ctx: ModelContext,
         trace: Trace,
         runtime: Runtime,
@@ -41,109 +62,71 @@ class ACP:
         secret: str,
         mcp_urls: dict[str, str],
         data: TaskData,
-        *,
-        env: dict[str, str],
-        command: list[str],
-        prompt: str | Messages | None,
-        system_prompt: str | None = None,
-        session_meta: dict | None = None,
-    ) -> "ACPHarnessSession":
-        """Create a persistent ACP-backed handle owned by one rollout."""
+    ) -> ACPConfig:
+        pass
+
+    async def session(
+        self,
+        ctx: ModelContext,
+        trace: Trace,
+        runtime: Runtime,
+        endpoint: str,
+        secret: str,
+        mcp_urls: dict[str, str],
+        data: TaskData,
+    ) -> HarnessSession:
+        if not runtime.supports_live_processes:
+            raise HarnessError(
+                f"harness {self.config.id!r} requires a runtime with live process support"
+            )
+        config = await self.prepare_acp(
+            ctx, trace, runtime, endpoint, secret, mcp_urls, data
+        )
         return ACPHarnessSession(
-            harness,
+            self,
             ctx,
             trace,
             runtime,
             endpoint,
             secret,
-            mcp_urls,
+            mcp_urls if config.mcp_urls is None else config.mcp_urls,
             data,
-            env=env,
-            command=command,
-            prompt=prompt,
-            system_prompt=system_prompt,
-            session_meta=session_meta,
+            config,
         )
 
-    async def run(
-        self,
-        runtime: Runtime,
-        env: dict[str, str],
-        command: list[str],
-        prompt: str | Messages | None,
-        *,
-        mcp_urls: dict[str, str] | None = None,
-        system_prompt: str | None = None,
-        session_path: str | None = None,
-        session_meta: dict | None = None,
-        allow_empty_tool_reply: bool = False,
-    ) -> ProgramResult:
-        """Run one ACP segment without retaining its process."""
-        return await self._run(
-            runtime,
-            env,
-            command,
-            prompt,
-            mcp_urls=mcp_urls,
-            system_prompt=system_prompt,
-            session_path=session_path,
-            session_meta=session_meta,
-            allow_empty_tool_reply=allow_empty_tool_reply,
-        )
-
-    async def _run(
+    async def launch(
         self,
+        ctx: ModelContext,
+        trace: Trace,
         runtime: Runtime,
-        env: dict[str, str],
-        command: list[str],
-        prompt: str | Messages | None,
-        *,
-        mcp_urls: dict[str, str] | None = None,
-        system_prompt: str | None = None,
-        session_path: str | None = None,
-        session_meta: dict | None = None,
-        allow_empty_tool_reply: bool = False,
+        endpoint: str,
+        secret: str,
+        mcp_urls: dict[str, str],
+        data: TaskData,
     ) -> ProgramResult:
-        if prompt is None:
-            raise ValueError("ACP requires a prompt")
-        messages = (
-            [{"role": "user", "content": prompt}]
-            if isinstance(prompt, str)
-            else [message_to_wire(message) for message in prompt]
-        )
-        config = {
-            "command": command,
-            "messages": messages,
-            "mcp_urls": mcp_urls or {},
-            "system_prompt": system_prompt or "",
-            "session_path": session_path,
-            "session_meta": session_meta or {},
-            "allow_empty_tool_reply": allow_empty_tool_reply,
-        }
-        program = await runtime.prepare_uv_script(
-            ACP_SOURCE,
-            {**env, "UV_FROZEN": "false"},
-            activate=False,
+        raise HarnessError(
+            f"harness {self.config.id!r} requires a rollout-scoped session"
         )
-        directory = f".vf-acp-{secrets.token_hex(8)}"
-        created = await runtime.run(["mkdir", "-m", "700", directory], {})
-        if created.exit_code != 0:
-            raise RuntimeError(f"ACP config directory failed: {created.stderr.strip()}")
-        path = f"{directory}/config.json"
-        try:
-            await runtime.write(path, json.dumps(config).encode())
-            return await runtime.run_program([*program, "once", path], env)
-        finally:
-            await run_shielded(runtime.run(["rm", "-rf", directory], {}))
 
 
-def _packet(value: dict) -> bytes:
+def _packet(value: JsonObject) -> bytes:
     data = json.dumps(value, ensure_ascii=False).encode()
     if len(data) > MAX_PACKET_BYTES:
         raise ValueError(f"ACP session packet is too large: {len(data)} bytes")
     return len(data).to_bytes(8, "big") + data
 
 
+def _require_model_turn(trace: Trace, calls_before: int, result: ProgramResult) -> None:
+    if (
+        result.exit_code
+        or trace.stop_condition is not None
+        or any(call.node is not None for call in trace.calls[calls_before:])
+    ):
+        return
+    detail = (result.stderr or result.stdout).strip()[-500:] or "<no output>"
+    raise RuntimeError("ACP agent completed without committing a model turn: " + detail)
+
+
 class _PacketReader:
     def __init__(self, source: AsyncIterator[bytes]) -> None:
         self._source = source.__aiter__()
@@ -159,7 +142,7 @@ class _PacketReader:
         del self._buffer[:size]
         return data
 
-    async def read(self) -> dict:
+    async def read(self) -> JsonObject:
         size = int.from_bytes(await self._readexactly(8), "big")
         if size > MAX_PACKET_BYTES:
             raise ValueError(f"ACP session packet is too large: {size} bytes")
@@ -179,18 +162,10 @@ class ACPHarnessSession(HarnessSession):
         secret: str,
         mcp_urls: dict[str, str],
         data: TaskData,
-        env: dict[str, str],
-        command: list[str],
-        prompt: str | Messages | None,
-        system_prompt: str | None,
-        session_meta: dict | None,
+        config: ACPConfig,
     ) -> None:
         super().__init__(harness, ctx, trace, runtime, endpoint, secret, mcp_urls, data)
-        self.env = env
-        self.command = command
-        self.prompt = prompt
-        self.system_prompt = system_prompt
-        self.session_meta = session_meta or {}
+        self.config = config
         self._process: RuntimeProcess | None = None
         self._reader: _PacketReader | None = None
         self._stderr_tail = bytearray()
@@ -201,10 +176,10 @@ class ACPHarnessSession(HarnessSession):
         self._stderr_tail.clear()
         program = await self.runtime.prepare_uv_script(
             ACP_SOURCE,
-            {**self.env, "UV_FROZEN": "false"},
+            {**self.config.env, "UV_FROZEN": "false"},
             activate=False,
         )
-        process = await self.runtime.open_process([*program, "stream"], self.env)
+        process = await self.runtime.open_process(program, self.config.env)
         self._process = process
         self._reader = _PacketReader(process.stdout)
         self._stderr_task = asyncio.create_task(self._drain_stderr(process.stderr))
@@ -219,21 +194,28 @@ class ACPHarnessSession(HarnessSession):
         return self._stderr_tail.decode(errors="replace").strip()
 
     async def _run(self, messages: Messages | None) -> ProgramResult:
-        prompt = self.prompt if messages is None else messages
+        prompt = self.config.prompt if messages is None else messages
         if prompt is None:
             raise ValueError("ACP requires a prompt")
-        wire_messages = (
-            [{"role": "user", "content": prompt}]
+        if not isinstance(prompt, str) and (
+            not prompt or any(message.role != "user" for message in prompt)
+        ):
+            raise ValueError("an ACP turn must contain user messages only")
+        user_contents = (
+            [prompt]
             if isinstance(prompt, str)
-            else [message_to_wire(message) for message in prompt]
+            else [
+                message.model_dump(mode="json", include={"content"})["content"]
+                for message in prompt
+            ]
         )
         config = {
-            "command": self.command,
-            "messages": wire_messages,
+            "command": self.config.command,
+            "user_contents": user_contents,
             "mcp_urls": self.mcp_urls,
-            "system_prompt": self.system_prompt or "",
-            "session_path": None,
-            "session_meta": self.session_meta,
+            "system_prompt": self.config.system_prompt or "",
+            "session_meta": self.config.session_meta or {},
+            "allow_empty_tool_reply": self.config.allow_empty_tool_reply,
         }
         async with self._lock:
             if self._closed:
@@ -244,6 +226,7 @@ class ACPHarnessSession(HarnessSession):
                 await self._start()
             assert self._process is not None
             assert self._reader is not None
+            calls_before = len(self.trace.calls)
             try:
                 await self._process.write(
                     _packet({"operation": "prompt", "config": config})
@@ -257,7 +240,12 @@ class ACPHarnessSession(HarnessSession):
             if stderr := self._stderr():
                 detail = f"{detail}\n\nACP process stderr:\n{stderr}"
             raise RuntimeError(detail)
-        return ProgramResult(exit_code=0, stdout=response.get("reply", ""), stderr="")
+        reply = response.get("reply", "")
+        if not isinstance(reply, str):
+            raise TypeError("ACP session reply must be a string")
+        result = ProgramResult(exit_code=0, stdout=reply, stderr="")
+        _require_model_turn(self.trace, calls_before, result)
+        return result
 
     async def _stop(self, *, graceful: bool) -> None:
         process, self._process = self._process, None
@@ -265,41 +253,30 @@ class ACPHarnessSession(HarnessSession):
         stderr_task, self._stderr_task = self._stderr_task, None
         if process is None:
             return
-        failure: BaseException | None = None
         try:
             if graceful and reader is not None:
-                try:
+                with contextlib.suppress(BaseException):
                     await process.write(_packet({"operation": "shutdown"}))
-                    response = await asyncio.wait_for(reader.read(), timeout=10)
-                    if not response.get("ok"):
-                        raise RuntimeError(
-                            response.get("error") or "ACP session shutdown failed"
-                        )
-                except BaseException as error:  # noqa: BLE001 - finish teardown if cancelled
-                    failure = error
-            try:
-                await asyncio.wait_for(process.wait(), timeout=10 if graceful else 0.1)
-            except BaseException:  # noqa: BLE001 - cancellation still requires termination
-                with contextlib.suppress(Exception):
-                    await asyncio.wait_for(process.terminate(), timeout=5)
-                try:
-                    await asyncio.wait_for(process.wait(), timeout=5)
-                except BaseException:  # noqa: BLE001 - cancellation still requires a kill
-                    with contextlib.suppress(Exception):
-                        await asyncio.wait_for(process.kill(), timeout=5)
+                    await asyncio.wait_for(reader.read(), timeout=10)
+            for timeout, stop in (
+                (10 if graceful else 0.1, None),
+                (5, process.terminate),
+                (5, process.kill),
+            ):
+                if stop is not None:
                     with contextlib.suppress(BaseException):
-                        await asyncio.wait_for(process.wait(), timeout=5)
+                        await stop()
+                try:
+                    await asyncio.wait_for(process.wait(), timeout)
+                    break
+                except TimeoutError:
+                    continue
         finally:
             if stderr_task is not None:
                 if not stderr_task.done():
                     stderr_task.cancel()
                 with contextlib.suppress(BaseException):
                     await stderr_task
-        if failure is not None:
-            detail = str(failure)
-            if stderr := self._stderr():
-                detail = f"{detail}\n\nACP process stderr:\n{stderr}"
-            raise RuntimeError(detail) from failure
 
     async def close(self) -> None:
         if self._closed: