verifiers 0.2.1.dev62__tar.gz → 0.2.1.dev64__tar.gz

{verifiers-0.2.1.dev62 → verifiers-0.2.1.dev64}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.1.dev62
+Version: 0.2.1.dev64
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.2.1.dev62 → verifiers-0.2.1.dev64}/verifiers/v1/harnesses/__init__.py

@@ -11,6 +11,7 @@ from verifiers.v1.harnesses.mini_swe_agent import (
 )
 from verifiers.v1.harnesses.null import NullHarness, NullHarnessConfig
 from verifiers.v1.harnesses.pi import PiHarness, PiHarnessConfig
+from verifiers.v1.harnesses.pool import PoolHarness, PoolHarnessConfig
 from verifiers.v1.harnesses.rlm import RLMHarness, RLMHarnessConfig
 from verifiers.v1.harnesses.terminus_2 import Terminus2Harness, Terminus2HarnessConfig
 
@@ -27,6 +28,8 @@ __all__ = [
     "NullHarnessConfig",
     "PiHarness",
     "PiHarnessConfig",
+    "PoolHarness",
+    "PoolHarnessConfig",
     "MiniSWEAgentHarness",
     "MiniSWEAgentHarnessConfig",
     "RLMHarness",

{verifiers-0.2.1.dev62 → verifiers-0.2.1.dev64}/verifiers/v1/harnesses/bash/program.py

@@ -1,6 +1,6 @@
 # /// script
 # requires-python = ">=3.10"
-# dependencies = ["openai", "mcp", "httpx"]
+# dependencies = ["openai", "mcp", "httpx", "tenacity"]
 # ///
 """Secrets arrive through argv so local tool subprocesses do not inherit them."""
 
@@ -8,14 +8,19 @@ import argparse
 import asyncio
 import json
 import subprocess
-from contextlib import AsyncExitStack
+from contextlib import AsyncExitStack, asynccontextmanager, suppress
 from pathlib import Path
 
 import httpx
 from openai import AsyncOpenAI
+from tenacity import AsyncRetrying, stop_after_attempt, wait_exponential_jitter
 
 SERPER_URL = "https://google.serper.dev/search"
 
+MCP_CALL_ATTEMPTS = 6
+MCP_TIMEOUT = httpx.Timeout(600.0, connect=5.0)  # the OpenAI SDK client defaults
+
+
 BASH_TOOL = {
     "type": "function",
     "function": {
@@ -177,29 +182,66 @@ async def chat(
     return completion.choices[0].message
 
 
-async def connect_mcp(
-    stack: AsyncExitStack, config: dict, reserved: set[str]
-) -> tuple[list[dict], dict]:
-    """Connect to each configured MCP server (a streamable-HTTP `url`); return
-    (tool schemas, dispatch mapping `<server>_<tool>` -> (session, raw tool name))."""
+@asynccontextmanager
+async def mcp_session(spec: dict):
+    """One fresh streamable-HTTP session to an MCP server, opened and closed within the caller's
+    task so AnyIO cancellation scopes stay correctly nested. A teardown failure after the body
+    completed is swallowed — the result is already in hand, and closing noise must not fail (or
+    replay) an already-answered call."""
     from mcp import ClientSession
     from mcp.client.streamable_http import (
         create_mcp_http_client,
         streamable_http_client,
     )
 
-    tool_schemas: list[dict] = []
-    dispatch: dict[str, tuple] = {}
-    for name, spec in config.get("mcpServers", {}).items():
+    stack = AsyncExitStack()
+    try:
         http_client = await stack.enter_async_context(
-            create_mcp_http_client(headers=spec.get("headers") or None)
+            create_mcp_http_client(
+                headers=spec.get("headers") or None, timeout=MCP_TIMEOUT
+            )
         )
         read, write, *_ = await stack.enter_async_context(
             streamable_http_client(spec["url"], http_client=http_client)
         )
         session = await stack.enter_async_context(ClientSession(read, write))
         await session.initialize()
-        for tool in (await session.list_tools()).tools:
+        yield session
+    finally:
+        with suppress(Exception):
+            await stack.aclose()
+
+
+async def with_retry(call):
+    """Run one session-scoped operation, retrying transient failures with backoff. A call whose
+    response was lost may be replayed — MCP has no idempotency key, so tools should tolerate
+    at-least-once delivery (a tool that fails reports through its result, not an exception)."""
+    async for attempt in AsyncRetrying(
+        stop=stop_after_attempt(MCP_CALL_ATTEMPTS),
+        wait=wait_exponential_jitter(initial=0.5, max=30),
+        reraise=True,
+    ):
+        with attempt:
+            return await call()
+
+
+async def connect_mcp(
+    config: dict, reserved: set[str]
+) -> tuple[list[dict], dict, dict]:
+    """Enumerate each configured MCP server's tools (a streamable-HTTP `url`); return (tool schemas,
+    dispatch mapping `<server>_<tool>` -> (server name, raw tool name), servers mapping name -> spec).
+    No session is held — a stateless-HTTP server is reconnected per call."""
+    tool_schemas: list[dict] = []
+    dispatch: dict[str, tuple] = {}
+    servers: dict[str, dict] = {}
+    for name, spec in config.get("mcpServers", {}).items():
+        servers[name] = spec
+
+        async def list_tools(spec: dict = spec):
+            async with mcp_session(spec) as session:
+                return (await session.list_tools()).tools
+
+        for tool in await with_retry(list_tools):
             # A server named "" (TOOL_PREFIX = None) advertises its tools bare.
             full = f"{name}_{tool.name}" if name else tool.name
             if full in reserved or full in dispatch:
@@ -216,8 +258,8 @@ async def connect_mcp(
                     },
                 }
             )
-            dispatch[full] = (session, tool.name)
-    return tool_schemas, dispatch
+            dispatch[full] = (name, tool.name)
+    return tool_schemas, dispatch, servers
 
 
 def mcp_content_to_chat_content(blocks) -> str | list[dict]:
@@ -237,9 +279,18 @@ def mcp_content_to_chat_content(blocks) -> str | list[dict]:
     return parts
 
 
-async def call_mcp(dispatch: dict, name: str, arguments: dict) -> str | list[dict]:
-    session, raw = dispatch[name]
-    result = await session.call_tool(raw, arguments)
+async def call_mcp(
+    servers: dict, dispatch: dict, name: str, arguments: dict
+) -> str | list[dict]:
+    """Call a tool on a fresh session per attempt — see `with_retry` for the replay semantics.
+    The result is converted outside the retry so a conversion failure fails once."""
+    server_name, raw = dispatch[name]
+
+    async def call():
+        async with mcp_session(servers[server_name]) as session:
+            return await session.call_tool(raw, arguments)
+
+    result = await with_retry(call)
     return mcp_content_to_chat_content(result.content)
 
 
@@ -268,84 +319,83 @@ async def main() -> None:
         initial = json.loads(payload)
     client = AsyncOpenAI(base_url=args.base_url, api_key=args.api_key)
     config = json.loads(args.mcp_config or "{}")
-    async with AsyncExitStack() as stack:
-        tools = [BASH_TOOL]
-        reserved = {"bash"}
-        if args.edit:
-            tools.append(EDIT_TOOL)
-            reserved.add("edit")
-        if args.search:
-            tools.append(SEARCH_TOOL)
-            reserved.add("search")
-        mcp_tools, dispatch = (
-            await connect_mcp(stack, config, reserved)
-            if config.get("mcpServers")
-            else ([], {})
-        )
-        tools += mcp_tools
-        messages = (
-            [{"role": "system", "content": args.system_prompt}]
-            if args.system_prompt
-            else []
-        )
-        if initial:
-            messages.extend(initial)
-        elif args.prompt:
-            messages.append({"role": "user", "content": args.prompt})
-        while True:
-            message = await chat(client, args.model, messages, tools)
-            messages.append(message.model_dump(exclude_none=True))
-            if not message.tool_calls:
-                break
-            for call in message.tool_calls:
-                name = call.function.name
-                try:
-                    tool_args = json.loads(call.function.arguments or "{}")
-                except json.JSONDecodeError as e:
-                    messages.append(
-                        {
-                            "role": "tool",
-                            "tool_call_id": call.id,
-                            "content": f"error: invalid JSON in tool arguments ({e}); resend the call with valid JSON",
-                        }
-                    )
-                    continue
-                # Valid JSON can still be a non-object (`[]`, `42`, `null`); the `.get(...)` calls
-                # below assume a dict, so reject anything else as a tool error rather than crashing.
-                if not isinstance(tool_args, dict):
-                    messages.append(
-                        {
-                            "role": "tool",
-                            "tool_call_id": call.id,
-                            "content": f"error: tool arguments must be a JSON object, got {type(tool_args).__name__}; resend as an object",
-                        }
-                    )
-                    continue
-                if name in dispatch:
-                    content = await call_mcp(dispatch, name, tool_args)
-                elif name == "bash":
-                    content = await asyncio.to_thread(
-                        run_bash, tool_args.get("command", "")
-                    )
-                elif name == "edit" and args.edit:
-                    content = await asyncio.to_thread(
-                        run_edit,
-                        tool_args.get("path"),
-                        tool_args.get("old_str"),
-                        tool_args.get("new_str"),
-                    )
-                elif name == "search" and args.search:
-                    content = await asyncio.to_thread(
-                        run_search,
-                        tool_args.get("query", ""),
-                        args.serper_key,
-                        tool_args.get("num_results", 5),
-                    )
-                else:
-                    content = f"error: unknown tool {name!r}"
+    tools = [BASH_TOOL]
+    reserved = {"bash"}
+    if args.edit:
+        tools.append(EDIT_TOOL)
+        reserved.add("edit")
+    if args.search:
+        tools.append(SEARCH_TOOL)
+        reserved.add("search")
+    mcp_tools, dispatch, servers = (
+        await connect_mcp(config, reserved)
+        if config.get("mcpServers")
+        else ([], {}, {})
+    )
+    tools += mcp_tools
+    messages = (
+        [{"role": "system", "content": args.system_prompt}]
+        if args.system_prompt
+        else []
+    )
+    if initial:
+        messages.extend(initial)
+    elif args.prompt:
+        messages.append({"role": "user", "content": args.prompt})
+    while True:
+        message = await chat(client, args.model, messages, tools)
+        messages.append(message.model_dump(exclude_none=True))
+        if not message.tool_calls:
+            break
+        for call in message.tool_calls:
+            name = call.function.name
+            try:
+                tool_args = json.loads(call.function.arguments or "{}")
+            except json.JSONDecodeError as e:
+                messages.append(
+                    {
+                        "role": "tool",
+                        "tool_call_id": call.id,
+                        "content": f"error: invalid JSON in tool arguments ({e}); resend the call with valid JSON",
+                    }
+                )
+                continue
+            # Valid JSON can still be a non-object (`[]`, `42`, `null`); the `.get(...)` calls
+            # below assume a dict, so reject anything else as a tool error rather than crashing.
+            if not isinstance(tool_args, dict):
                 messages.append(
-                    {"role": "tool", "tool_call_id": call.id, "content": content}
+                    {
+                        "role": "tool",
+                        "tool_call_id": call.id,
+                        "content": f"error: tool arguments must be a JSON object, got {type(tool_args).__name__}; resend as an object",
+                    }
+                )
+                continue
+            if name in dispatch:
+                content = await call_mcp(servers, dispatch, name, tool_args)
+            elif name == "bash":
+                content = await asyncio.to_thread(
+                    run_bash, tool_args.get("command", "")
                 )
+            elif name == "edit" and args.edit:
+                content = await asyncio.to_thread(
+                    run_edit,
+                    tool_args.get("path"),
+                    tool_args.get("old_str"),
+                    tool_args.get("new_str"),
+                )
+            elif name == "search" and args.search:
+                content = await asyncio.to_thread(
+                    run_search,
+                    tool_args.get("query", ""),
+                    args.serper_key,
+                    tool_args.get("num_results", 5),
+                )
+            else:
+                content = f"error: unknown tool {name!r}"
+            messages.append(
+                {"role": "tool", "tool_call_id": call.id, "content": content}
+            )
 
 
 if __name__ == "__main__":

verifiers-0.2.1.dev64/verifiers/v1/harnesses/null/program.py

@@ -0,0 +1,217 @@
+# /// script
+# requires-python = ">=3.11"
+# dependencies = ["openai", "mcp", "httpx", "tenacity"]
+# ///
+"""The interception endpoint and secret arrive through argv rather than the environment."""
+
+import argparse
+import asyncio
+import json
+from contextlib import AsyncExitStack, asynccontextmanager, suppress
+from pathlib import Path
+
+import httpx
+from openai import AsyncOpenAI
+from tenacity import AsyncRetrying, stop_after_attempt, wait_exponential_jitter
+
+MCP_CALL_ATTEMPTS = 6
+MCP_TIMEOUT = httpx.Timeout(600.0, connect=5.0)  # the OpenAI SDK client defaults
+
+
+async def chat(
+    client: AsyncOpenAI, model: str, messages: list[dict], tools: list[dict]
+):
+    completion = await client.chat.completions.create(
+        model=model, messages=messages, tools=tools or None
+    )
+    return completion.choices[0].message
+
+
+@asynccontextmanager
+async def mcp_session(spec: dict):
+    """One fresh streamable-HTTP session to an MCP server, opened and closed within the caller's
+    task so AnyIO cancellation scopes stay correctly nested. A teardown failure after the body
+    completed is swallowed — the result is already in hand, and closing noise must not fail (or
+    replay) an already-answered call."""
+    from mcp import ClientSession
+    from mcp.client.streamable_http import (
+        create_mcp_http_client,
+        streamable_http_client,
+    )
+
+    stack = AsyncExitStack()
+    try:
+        http_client = await stack.enter_async_context(
+            create_mcp_http_client(
+                headers=spec.get("headers") or None, timeout=MCP_TIMEOUT
+            )
+        )
+        read, write, *_ = await stack.enter_async_context(
+            streamable_http_client(spec["url"], http_client=http_client)
+        )
+        session = await stack.enter_async_context(ClientSession(read, write))
+        await session.initialize()
+        yield session
+    finally:
+        with suppress(Exception):
+            await stack.aclose()
+
+
+async def with_retry(call):
+    """Run one session-scoped operation, retrying transient failures with backoff. A call whose
+    response was lost may be replayed — MCP has no idempotency key, so tools should tolerate
+    at-least-once delivery (a tool that fails reports through its result, not an exception)."""
+    async for attempt in AsyncRetrying(
+        stop=stop_after_attempt(MCP_CALL_ATTEMPTS),
+        wait=wait_exponential_jitter(initial=0.5, max=30),
+        reraise=True,
+    ):
+        with attempt:
+            return await call()
+
+
+async def connect_mcp(config: dict) -> tuple[list[dict], dict, dict]:
+    """Enumerate each configured MCP server's tools (a streamable-HTTP `url`); return (tool schemas,
+    dispatch mapping advertised name -> (server name, raw tool name), servers mapping name -> spec).
+    No session is held — a stateless-HTTP server is reconnected per call. Tools are advertised as
+    `<server>_<tool>`; a server named `""` (TOOL_PREFIX = None) advertises its tools bare, so names
+    must be unique across the rollout's servers."""
+    tool_schemas: list[dict] = []
+    dispatch: dict[str, tuple] = {}
+    servers: dict[str, dict] = {}
+    for name, spec in config.get("mcpServers", {}).items():
+        servers[name] = spec
+
+        async def list_tools(spec: dict = spec):
+            async with mcp_session(spec) as session:
+                return (await session.list_tools()).tools
+
+        for tool in await with_retry(list_tools):
+            full = f"{name}_{tool.name}" if name else tool.name
+            if full in dispatch:
+                raise ValueError(
+                    f"duplicate tool name {full!r} across servers; keep qualified names"
+                )
+            tool_schemas.append(
+                {
+                    "type": "function",
+                    "function": {
+                        "name": full,
+                        "description": tool.description or "",
+                        "parameters": tool.inputSchema,
+                    },
+                }
+            )
+            dispatch[full] = (name, tool.name)
+    return tool_schemas, dispatch, servers
+
+
+def mcp_content_to_chat_content(blocks) -> str | list[dict]:
+    parts = []
+    for block in blocks:
+        if block.type == "text":
+            parts.append({"type": "text", "text": block.text})
+        elif block.type == "image":
+            url = f"data:{block.mimeType};base64,{block.data}"
+            parts.append({"type": "image_url", "image_url": {"url": url}})
+        else:
+            parts.append({"type": "text", "text": str(block)})
+    if not parts:
+        return str(blocks)
+    if all(part["type"] == "text" for part in parts):
+        return "\n".join(part["text"] for part in parts)
+    return parts
+
+
+async def call_mcp(
+    servers: dict, dispatch: dict, name: str, arguments: dict
+) -> str | list[dict]:
+    """Call a tool on a fresh session per attempt — see `with_retry` for the replay semantics.
+    The result is converted outside the retry so a conversion failure fails once."""
+    server_name, raw = dispatch[name]
+
+    async def call():
+        async with mcp_session(servers[server_name]) as session:
+            return await session.call_tool(raw, arguments)
+
+    result = await with_retry(call)
+    return mcp_content_to_chat_content(result.content)
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--base-url", required=True)
+    parser.add_argument("--api-key", required=True)
+    parser.add_argument("--model", required=True)
+    parser.add_argument("--system-prompt", default="")
+    parser.add_argument("--prompt", default="")
+    parser.add_argument("--initial-messages-file", default="")
+    parser.add_argument("--mcp-config", default="")
+    return parser.parse_args()
+
+
+async def main() -> None:
+    args = parse_args()
+    initial = []
+    if args.initial_messages_file:
+        path = Path(args.initial_messages_file)
+        payload = path.read_bytes()
+        path.unlink()
+        initial = json.loads(payload)
+    client = AsyncOpenAI(base_url=args.base_url, api_key=args.api_key)
+    config = json.loads(args.mcp_config or "{}")
+    if config.get("mcpServers"):
+        # Bound only tool enumeration; each session is opened and closed within this task.
+        async with asyncio.timeout(60):
+            tools, dispatch, servers = await connect_mcp(config)
+    else:
+        tools, dispatch, servers = [], {}, {}
+    messages = (
+        [{"role": "system", "content": args.system_prompt}]
+        if args.system_prompt
+        else []
+    )
+    if initial:
+        messages.extend(initial)
+    elif args.prompt:
+        messages.append({"role": "user", "content": args.prompt})
+    while True:
+        message = await chat(client, args.model, messages, tools)
+        messages.append(message.model_dump(exclude_none=True))
+        if not message.tool_calls:
+            break
+        for call in message.tool_calls:
+            name = call.function.name
+            try:
+                tool_args = json.loads(call.function.arguments or "{}")
+            except json.JSONDecodeError as e:
+                messages.append(
+                    {
+                        "role": "tool",
+                        "tool_call_id": call.id,
+                        "content": f"error: invalid JSON in tool arguments ({e}); resend the call with valid JSON",
+                    }
+                )
+                continue
+            # Valid JSON can still be a non-object (`[]`, `42`, `null`); the MCP dispatch
+            # assumes a dict, so reject anything else as a tool error rather than crashing.
+            if not isinstance(tool_args, dict):
+                messages.append(
+                    {
+                        "role": "tool",
+                        "tool_call_id": call.id,
+                        "content": f"error: tool arguments must be a JSON object, got {type(tool_args).__name__}; resend as an object",
+                    }
+                )
+                continue
+            if name in dispatch:
+                content = await call_mcp(servers, dispatch, name, tool_args)
+            else:
+                content = f"error: unknown tool {name!r}"
+            messages.append(
+                {"role": "tool", "tool_call_id": call.id, "content": content}
+            )
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

verifiers-0.2.1.dev64/verifiers/v1/harnesses/pool/__init__.py

@@ -0,0 +1,3 @@
+from verifiers.v1.harnesses.pool.harness import PoolHarness, PoolHarnessConfig
+
+__all__ = ["PoolHarness", "PoolHarnessConfig"]

verifiers-0.2.1.dev64/verifiers/v1/harnesses/pool/harness.py

@@ -0,0 +1,130 @@
+"""Run Poolside's `pool exec` against interception as an OpenAI-compatible provider."""
+
+import json
+import shlex
+
+from pydantic import Field
+
+from verifiers.v1.clients import ModelContext
+from verifiers.v1.harness import Harness, HarnessConfig
+from verifiers.v1.runtimes import ProgramResult, Runtime
+from verifiers.v1.trace import Trace
+from verifiers.v1.types import SystemMessage, TextContentPart, UserMessage
+
+POOL_DIR = "/tmp/vf-pool-{version}"
+SETTINGS_PATH = ".poolside/settings.local.yaml"
+INSTALL = r"""
+set -e
+command -v curl >/dev/null || (apt-get update -qq && apt-get install -y -qq curl ca-certificates >/dev/null)
+command -v tar >/dev/null || (apt-get update -qq && apt-get install -y -qq tar >/dev/null)
+case "$(uname -s)" in Linux) os=linux ;; Darwin) os=darwin ;; *) echo "unsupported os: $(uname -s)" >&2; exit 1 ;; esac
+case "$(uname -m)" in aarch64|arm64) arch=arm64 ;; x86_64|amd64) arch=amd64 ;; *) echo "unsupported arch: $(uname -m)" >&2; exit 1 ;; esac
+mkdir -p {dir}
+curl -fsSL "https://github.com/poolsideai/pool/releases/download/v{version}/pool-$os-$arch.tar.gz" | tar -xz -C {dir}
+mv "{dir}/pool-$os-$arch" "{dir}/pool"
+chmod +x "{dir}/pool"
+"""
+
+
+class PoolHarnessConfig(HarnessConfig):
+    version: str = Field(default="1.0.11", pattern=r"^[A-Za-z0-9._+-]+$")
+    """Pool release to install, pinned for reproducibility."""
+
+
+class PoolHarness(Harness[PoolHarnessConfig]):
+    APPENDS_SYSTEM_PROMPT = True
+    SUPPORTS_MCP = True
+    SUPPORTS_MESSAGE_PROMPT = True
+
+    async def setup(self, runtime: Runtime) -> None:
+        directory = POOL_DIR.format(version=self.config.version)
+        binary = f"{directory}/pool"
+        script = INSTALL.replace("{version}", self.config.version).replace(
+            "{dir}", directory
+        )
+        ensure = shlex.quote(f"[ -x {binary} ] || ({script})")
+        # Cache the pinned binary across local rollouts; Linux has flock, macOS has lockf.
+        guarded = (
+            f"mkdir -p {directory} && "
+            f'"$(command -v flock || command -v lockf)" {directory}/install.lock '
+            f"bash -o pipefail -c {ensure}"
+        )
+        result = await runtime.run(["sh", "-c", guarded], self.config.resolved_env)
+        if result.exit_code != 0:
+            detail = (result.stderr or result.stdout).strip()[-500:]
+            raise RuntimeError(f"Pool install failed: {detail}")
+
+    async def launch(
+        self,
+        ctx: ModelContext,
+        trace: Trace,
+        runtime: Runtime,
+        endpoint: str,
+        secret: str,
+        mcp_urls: dict[str, str],
+    ) -> ProgramResult:
+        system_prompt, prompt = self.resolve_prompt(trace.task.data)
+        if prompt is None:
+            raise ValueError("Pool requires a task prompt (it has no user simulator)")
+        texts = [system_prompt] if system_prompt else []
+        if isinstance(prompt, str):
+            texts.append(prompt)
+        else:
+            for message in prompt:
+                if not isinstance(message, (SystemMessage, UserMessage)):
+                    raise ValueError(
+                        "pool exec only supports system and user initial messages"
+                    )
+                parts = (
+                    [TextContentPart(text=message.content)]
+                    if isinstance(message.content, str)
+                    else message.content
+                )
+                for part in parts:
+                    if not isinstance(part, TextContentPart):
+                        raise ValueError("pool exec does not support image prompts")
+                    texts.append(part.text)
+        text = "\n\n".join(text for text in texts if text)
+
+        settings = {
+            "mcp_servers": {
+                name: {"transport": {"type": "http", "url": url}}
+                for name, url in mcp_urls.items()
+            },
+            # Values are Pool tool names such as `shell`, `read`, or `edit`.
+            "tools": {
+                name: {"disabled": True} for name in self.config.disabled_tools or []
+            },
+        }
+        await runtime.write(SETTINGS_PATH, json.dumps(settings).encode())
+
+        prompt_path = f".vf-pool/prompt-{trace.id}.txt"
+        await runtime.write(prompt_path, text.encode())
+        env = {
+            **self.config.resolved_env,
+            # Standalone provider mode sends this bearer and model to interception.
+            "POOLSIDE_API_KEY": secret,
+            "POOLSIDE_STANDALONE_BASE_URL": endpoint,
+            "POOLSIDE_STANDALONE_MODEL": ctx.model,
+        }
+        directory = POOL_DIR.format(version=self.config.version)
+        argv = [
+            f"{directory}/pool",
+            "exec",
+            "--unsafe-auto-allow",
+            "--sandbox",
+            "disabled",
+            "--prompt-file",
+            prompt_path,
+        ]
+        # Keep Pool's home, config, logs, and trajectories inside the rollout workspace.
+        isolate = (
+            'export HOME="$PWD/.vf-pool/home" '
+            'XDG_CONFIG_HOME="$PWD/.vf-pool/config" '
+            'XDG_STATE_HOME="$PWD/.vf-pool/state"; exec "$@"'
+        )
+        result = await runtime.run_program(["sh", "-c", isolate, "pool", *argv], env)
+        # Exit 4 means the agent declined the task, not that the harness failed.
+        if result.exit_code == 4:
+            return ProgramResult(0, result.stdout, result.stderr)
+        return result