verifiers 0.2.2.dev18__py3-none-any.whl → 0.2.2.dev19__py3-none-any.whl

verifiers/v1/__init__.py

@@ -2,6 +2,7 @@ import logging as _logging
 
 from pydantic_config import BaseConfig
 
+from verifiers.v1.acp import ACP
 from verifiers.v1.clients import (
     BaseClientConfig,
     Client,
@@ -12,7 +13,14 @@ from verifiers.v1.clients import (
     resolve_client,
 )
 from verifiers.v1.decorators import metric, reward, stop, tool
-from verifiers.v1.agent import Agent, AgentConfig, Agents, make_agent
+from verifiers.v1.agent import (
+    Agent,
+    AgentConfig,
+    Agents,
+    Interaction,
+    Segment,
+    make_agent,
+)
 from verifiers.v1.configs.env import (
     ElasticPoolConfig,
     EnvServerConfig,
@@ -35,7 +43,6 @@ from verifiers.v1.errors import (
     TaskError,
     ToolsetError,
     TunnelError,
-    UserError,
 )
 from verifiers.v1.harness import Harness, HarnessConfig
 from verifiers.v1.judge import (
@@ -108,8 +115,6 @@ from verifiers.v1.mcp import (
     Toolset,
     SharedToolsetConfig,
     ToolsetConfig,
-    User,
-    UserConfig,
 )
 from verifiers.v1.graph import MessageNode
 from verifiers.v1.episode import Episode, WireEpisode
@@ -216,7 +221,6 @@ __all__ = [
     "ProviderError",
     "HarnessError",
     "ToolsetError",
-    "UserError",
     "SandboxError",
     "TaskError",
     "InterceptionError",
@@ -235,6 +239,7 @@ __all__ = [
     "BaseConfig",
     "Harness",
     "HarnessConfig",
+    "ACP",
     "ModelContext",
     "Runtime",
     "RuntimeConfig",
@@ -302,9 +307,9 @@ __all__ = [
     "Toolset",
     "SharedToolsetConfig",
     "ToolsetConfig",
-    # user simulator
-    "User",
-    "UserConfig",
+    # the user channel
+    "Interaction",
+    "Segment",
 ]
 
 # The library logs via stdlib logging (per-module `getLogger(__name__)`), but is

verifiers/v1/acp/__init__.py

@@ -0,0 +1,64 @@
+"""Public Agent Client Protocol support for harness programs."""
+
+import json
+import secrets
+from pathlib import Path
+
+from verifiers.v1.dialects.chat import message_to_wire
+from verifiers.v1.harness import Harness
+from verifiers.v1.runtimes import ProgramResult, Runtime
+from verifiers.v1.types import Messages
+from verifiers.v1.utils.aio import run_shielded
+
+ACP_SOURCE = (Path(__file__).resolve().parent / "_runner.py").read_text()
+
+__all__ = ["ACP"]
+
+
+class ACP:
+    """Run an ACP agent."""
+
+    async def setup(self, harness: Harness, runtime: Runtime) -> None:
+        await runtime.prepare_uv_script(
+            ACP_SOURCE, {**harness.config.resolved_env, "UV_FROZEN": "false"}
+        )
+
+    async def run(
+        self,
+        runtime: Runtime,
+        env: dict[str, str],
+        command: list[str],
+        prompt: str | Messages | None,
+        *,
+        mcp_urls: dict[str, str] | None = None,
+        system_prompt: str | None = None,
+        session_path: str | None = None,
+    ) -> ProgramResult:
+        if prompt is None:
+            raise ValueError("ACP requires a prompt")
+        messages = (
+            [{"role": "user", "content": prompt}]
+            if isinstance(prompt, str)
+            else [message_to_wire(message) for message in prompt]
+        )
+        config = {
+            "command": command,
+            "messages": messages,
+            "mcp_urls": mcp_urls or {},
+            "system_prompt": system_prompt or "",
+            "session_path": session_path,
+        }
+        program = await runtime.prepare_uv_script(
+            ACP_SOURCE, {**env, "UV_FROZEN": "false"}
+        )
+        directory = f".vf-acp-{secrets.token_hex(8)}"
+        created = await runtime.run(["mkdir", "-m", "700", directory], {})
+        if created.exit_code != 0:
+            raise RuntimeError(f"ACP config directory failed: {created.stderr.strip()}")
+        path = f"{directory}/config.json"
+        try:
+            await runtime.write(path, json.dumps(config).encode())
+            result = await runtime.run_program([*program, path], env)
+            return result
+        finally:
+            await run_shielded(runtime.run(["rm", "-rf", directory], {}))

verifiers/v1/acp/_runner.py

@@ -0,0 +1,192 @@
+# /// script
+# requires-python = ">=3.10,<3.15"
+# dependencies = ["agent-client-protocol==0.11.0"]
+# ///
+"""Run one harness segment through an ACP agent."""
+
+import asyncio
+import json
+import os
+import sys
+from pathlib import Path
+from typing import Any
+
+from acp import (
+    PROTOCOL_VERSION,
+    Client,
+    RequestError,
+    image_block,
+    spawn_agent_process,
+    text_block,
+)
+from acp.schema import (
+    AgentMessageChunk,
+    AllowedOutcome,
+    ClientCapabilities,
+    DeniedOutcome,
+    HttpMcpServer,
+    PermissionOption,
+    RequestPermissionResponse,
+    TextContentBlock,
+)
+
+
+class VerifiersClient(Client):
+    def __init__(self) -> None:
+        self.visible_reply = ""
+        self.message_id: str | None = None
+
+    async def session_update(self, session_id: str, update: Any, **kwargs: Any) -> None:
+        if not isinstance(update, AgentMessageChunk) or not isinstance(
+            update.content, TextContentBlock
+        ):
+            return
+        message_id = getattr(update, "message_id", None)
+        if message_id is not None and message_id != self.message_id:
+            self.visible_reply = ""
+            self.message_id = message_id
+        self.visible_reply += update.content.text
+
+    async def request_permission(
+        self,
+        session_id: str,
+        tool_call: Any,
+        options: list[PermissionOption],
+        **kwargs: Any,
+    ) -> RequestPermissionResponse:
+        option = next(
+            (item for item in options if item.kind in ("allow_once", "allow_always")),
+            None,
+        )
+        outcome = (
+            AllowedOutcome(outcome="selected", option_id=option.option_id)
+            if option
+            else DeniedOutcome(outcome="cancelled")
+        )
+        return RequestPermissionResponse(outcome=outcome)
+
+
+def content_blocks(messages: list[dict], supports_images: bool) -> list:
+    blocks = []
+    transcript = len(messages) != 1 or messages[0].get("role") != "user"
+    for message in messages:
+        if transcript:
+            separator = "\n\n" if blocks else ""
+            blocks.append(
+                text_block(f"{separator}[{message.get('role', 'message')}]\n")
+            )
+        content = message.get("content") or ""
+        parts = (
+            [{"type": "text", "text": content}] if isinstance(content, str) else content
+        )
+        for part in parts:
+            if part["type"] == "text":
+                blocks.append(text_block(part["text"]))
+                continue
+            if not supports_images:
+                raise ValueError("ACP agent does not support image prompts")
+            url = part["image_url"]["url"]
+            metadata, separator, data = url.partition(",")
+            media_type, *parameters = metadata.removeprefix("data:").split(";")
+            if (
+                not separator
+                or not metadata.startswith("data:image/")
+                or not any(value.lower() == "base64" for value in parameters)
+            ):
+                raise ValueError("ACP image prompts require base64 data:image URLs")
+            blocks.append(image_block(data, media_type))
+        metadata = {
+            key: value
+            for key, value in message.items()
+            if key not in ("role", "content") and value
+        }
+        if metadata:
+            blocks.append(text_block("\n" + json.dumps(metadata, ensure_ascii=False)))
+    return blocks
+
+
+async def run_client(config: dict) -> None:
+    client = VerifiersClient()
+    command = config["command"]
+    async with spawn_agent_process(
+        client,
+        command[0],
+        *command[1:],
+        env=os.environ.copy(),
+        transport_kwargs={"stderr": None},
+    ) as (connection, _process):
+        initialized = await connection.initialize(
+            protocol_version=PROTOCOL_VERSION,
+            client_capabilities=ClientCapabilities(),
+        )
+        capabilities = initialized.agent_capabilities
+        prompt_capabilities = capabilities and capabilities.prompt_capabilities
+        supports_images = bool(prompt_capabilities and prompt_capabilities.image)
+        mcp_servers = [
+            HttpMcpServer(type="http", name=name, url=url, headers=[])
+            for name, url in config["mcp_urls"].items()
+        ]
+        session_path = Path(config["session_path"]) if config["session_path"] else None
+        is_new = session_path is None or not session_path.exists()
+        if is_new:
+            session = await connection.new_session(
+                cwd=os.getcwd(), mcp_servers=mcp_servers
+            )
+            session_id = session.session_id
+        else:
+            session_id = session_path.read_text().strip()
+            session_capabilities = capabilities and capabilities.session_capabilities
+            if session_capabilities and session_capabilities.resume is not None:
+                await connection.resume_session(
+                    cwd=os.getcwd(), session_id=session_id, mcp_servers=mcp_servers
+                )
+            elif capabilities and capabilities.load_session:
+                await connection.load_session(
+                    cwd=os.getcwd(), session_id=session_id, mcp_servers=mcp_servers
+                )
+            else:
+                raise RuntimeError("ACP agent does not support resuming sessions")
+
+        messages = config["messages"]
+        if not is_new:
+            last_assistant = max(
+                (
+                    index
+                    for index, message in enumerate(messages)
+                    if message.get("role") == "assistant"
+                ),
+                default=-1,
+            )
+            messages = messages[last_assistant + 1 :]
+        if is_new and config["system_prompt"]:
+            messages = [
+                {"role": "system", "content": config["system_prompt"]},
+                *messages,
+            ]
+        prompt = content_blocks(messages, supports_images)
+        if not prompt:
+            raise ValueError("ACP prompt has no content")
+        client.visible_reply = ""
+        client.message_id = None
+        try:
+            await connection.prompt(session_id=session_id, prompt=prompt)
+        except RequestError as error:
+            detail = error.data.get("details") if isinstance(error.data, dict) else None
+            raise RuntimeError(detail or str(error)) from error
+        if not client.visible_reply.strip():
+            raise RuntimeError("ACP agent produced no visible reply")
+        sys.stdout.write(client.visible_reply)
+        if session_path and is_new:
+            session_path.parent.mkdir(parents=True, exist_ok=True)
+            session_path.write_text(session_id)
+
+
+async def main() -> None:
+    path = Path(sys.argv[1])
+    config = json.loads(path.read_text())
+    path.unlink()
+    await run_client(config)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())