verifiers 0.3.2.dev65__tar.gz → 0.3.2.dev67__tar.gz

{verifiers-0.3.2.dev65 → verifiers-0.3.2.dev67}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.3.2.dev65
+Version: 0.3.2.dev67
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.3.2.dev65 → verifiers-0.3.2.dev67}/verifiers/v1/harnesses/bash/harness.py

@@ -8,7 +8,6 @@ from verifiers.v1.configs.harness import HarnessConfig
 from verifiers.v1.harness import Harness
 from verifiers.v1.harnesses.utils.launch import (
     CHAT_PROGRAM_SOURCE,
-    MCP_CHAT_PROGRAM_SOURCE,
     launch_chat_program,
 )
 from verifiers.v1.runtimes import ProgramResult, Runtime
@@ -122,7 +121,6 @@ class BashHarness(Harness[BashHarnessConfig]):
             mcp_urls,
             system_prompt,
             prompt,
-            source_with_mcp=MCP_CHAT_PROGRAM_SOURCE,
             extra_args=args,
             env=env,
             activate=False,

{verifiers-0.3.2.dev65 → verifiers-0.3.2.dev67}/verifiers/v1/harnesses/browser_use/program.py

@@ -27,6 +27,7 @@ import subprocess
 import sys
 import time
 import urllib.request
+from contextlib import AsyncExitStack
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, cast
 from urllib.parse import urlsplit
@@ -226,63 +227,60 @@ async def main() -> None:
     config = json.loads(args.mcp_config or "{}")
     tools = [BROWSER_TOOL]
     reserved = {"browser"}
-    mcp_tools, dispatch, servers = (
-        await connect_mcp(config, reserved)
-        if config.get("mcpServers")
-        else ([], {}, {})
-    )
-    tools += mcp_tools
-    messages = (
-        [{"role": "system", "content": args.system_prompt}]
-        if args.system_prompt
-        else []
-    )
-    if initial:
-        messages.extend(initial)
-    elif args.prompt:
-        messages.append({"role": "user", "content": args.prompt})
-    while True:
-        message = await chat(client, args.model, messages, tools)
-        messages.append(message.model_dump(exclude_none=True))
-        if not message.tool_calls:
-            break
-        for call in message.tool_calls:
-            name = call.function.name
-            try:
-                tool_args = json.loads(call.function.arguments or "{}")
-            except json.JSONDecodeError as e:
-                messages.append(
-                    {
-                        "role": "tool",
-                        "tool_call_id": call.id,
-                        "content": f"error: invalid JSON in tool arguments ({e}); resend the call with valid JSON",
-                    }
-                )
-                continue
-            # Valid JSON can still be a non-object (`[]`, `42`, `null`); the `.get(...)` calls
-            # below assume a dict, so reject anything else as a tool error rather than crashing.
-            if not isinstance(tool_args, dict):
+    async with AsyncExitStack() as mcp_stack:
+        mcp_tools, dispatch, servers = await connect_mcp(config, mcp_stack, reserved)
+        tools += mcp_tools
+        messages = (
+            [{"role": "system", "content": args.system_prompt}]
+            if args.system_prompt
+            else []
+        )
+        if initial:
+            messages.extend(initial)
+        elif args.prompt:
+            messages.append({"role": "user", "content": args.prompt})
+        while True:
+            message = await chat(client, args.model, messages, tools)
+            messages.append(message.model_dump(exclude_none=True))
+            if not message.tool_calls:
+                break
+            for call in message.tool_calls:
+                name = call.function.name
+                try:
+                    tool_args = json.loads(call.function.arguments or "{}")
+                except json.JSONDecodeError as e:
+                    messages.append(
+                        {
+                            "role": "tool",
+                            "tool_call_id": call.id,
+                            "content": f"error: invalid JSON in tool arguments ({e}); resend the call with valid JSON",
+                        }
+                    )
+                    continue
+                # Valid JSON can still be a non-object (`[]`, `42`, `null`); the `.get(...)` calls
+                # below assume a dict, so reject anything else as a tool error rather than crashing.
+                if not isinstance(tool_args, dict):
+                    messages.append(
+                        {
+                            "role": "tool",
+                            "tool_call_id": call.id,
+                            "content": f"error: tool arguments must be a JSON object, got {type(tool_args).__name__}; resend as an object",
+                        }
+                    )
+                    continue
+                if name in dispatch:
+                    content = await call_mcp(servers, dispatch, name, tool_args)
+                elif name == "browser":
+                    content = await asyncio.to_thread(
+                        run_browser,
+                        tool_args.get("code", ""),
+                        browser_env,
+                    )
+                else:
+                    content = f"error: unknown tool {name!r}"
                 messages.append(
-                    {
-                        "role": "tool",
-                        "tool_call_id": call.id,
-                        "content": f"error: tool arguments must be a JSON object, got {type(tool_args).__name__}; resend as an object",
-                    }
-                )
-                continue
-            if name in dispatch:
-                content = await call_mcp(servers, dispatch, name, tool_args)
-            elif name == "browser":
-                content = await asyncio.to_thread(
-                    run_browser,
-                    tool_args.get("code", ""),
-                    browser_env,
+                    {"role": "tool", "tool_call_id": call.id, "content": content}
                 )
-            else:
-                content = f"error: unknown tool {name!r}"
-            messages.append(
-                {"role": "tool", "tool_call_id": call.id, "content": content}
-            )
 
 
 if __name__ == "__main__":

{verifiers-0.3.2.dev65 → verifiers-0.3.2.dev67}/verifiers/v1/harnesses/null/harness.py

@@ -3,7 +3,6 @@ from verifiers.v1.configs.harness import HarnessConfig
 from verifiers.v1.harness import Harness
 from verifiers.v1.harnesses.utils.launch import (
     CHAT_PROGRAM_SOURCE,
-    MCP_CHAT_PROGRAM_SOURCE,
     launch_chat_program,
 )
 from verifiers.v1.runtimes import ProgramResult, Runtime
@@ -47,5 +46,4 @@ class NullHarness(Harness[NullHarnessConfig]):
             mcp_urls,
             system_prompt,
             prompt,
-            source_with_mcp=MCP_CHAT_PROGRAM_SOURCE,
         )

{verifiers-0.3.2.dev65 → verifiers-0.3.2.dev67}/verifiers/v1/harnesses/utils/core.py

@@ -4,6 +4,7 @@ import argparse
 import asyncio
 import json
 import subprocess
+from contextlib import AsyncExitStack
 from pathlib import Path
 from typing import TYPE_CHECKING
 
@@ -361,35 +362,37 @@ async def main() -> None:
     if args.search:
         tools.append(SEARCH_TOOL)
         reserved.add("search")
-    if config.get("mcpServers"):
-        mcp_tools, dispatch, servers = await asyncio.wait_for(
-            connect_mcp(config, reserved), timeout=None if args.bash else 60
+    async with AsyncExitStack() as mcp_stack:
+        if config.get("mcpServers"):
+            mcp_tools, dispatch, servers = await asyncio.wait_for(
+                connect_mcp(config, mcp_stack, reserved),
+                timeout=None if args.bash else 60,
+            )
+        else:
+            mcp_tools, dispatch, servers = [], {}, {}
+        tools += mcp_tools
+        messages = (
+            [{"role": "system", "content": args.system_prompt}]
+            if args.system_prompt
+            else []
         )
-    else:
-        mcp_tools, dispatch, servers = [], {}, {}
-    tools += mcp_tools
-    messages = (
-        [{"role": "system", "content": args.system_prompt}]
-        if args.system_prompt
-        else []
-    )
-    if initial:
-        messages.extend(initial)
-    elif args.prompt:
-        messages.append({"role": "user", "content": args.prompt})
-    compactor = Compactor(
-        client,
-        args.model,
-        tools,
-        args.compaction,
-        args.summarize_at_tokens,
-    )
-    if compactor.enabled and compactor.threshold is None:
-        compactor.threshold = await discover_threshold(client, args.model)
-    # The initial conversation is the floor for checkpoint fallbacks: a first-turn
-    # checkpoint must never retry from an empty base.
-    compactor.note_good(messages)
-    await run_chat_loop(args, compactor, messages, dispatch, servers, tool_client)
+        if initial:
+            messages.extend(initial)
+        elif args.prompt:
+            messages.append({"role": "user", "content": args.prompt})
+        compactor = Compactor(
+            client,
+            args.model,
+            tools,
+            args.compaction,
+            args.summarize_at_tokens,
+        )
+        if compactor.enabled and compactor.threshold is None:
+            compactor.threshold = await discover_threshold(client, args.model)
+        # The initial conversation is the floor for checkpoint fallbacks: a first-turn
+        # checkpoint must never retry from an empty base.
+        compactor.note_good(messages)
+        await run_chat_loop(args, compactor, messages, dispatch, servers, tool_client)
     if tool_client is not None:
         await tool_client.aclose()
 

{verifiers-0.3.2.dev65 → verifiers-0.3.2.dev67}/verifiers/v1/harnesses/utils/launch.py

@@ -29,8 +29,7 @@ CHAT_PROGRAM = (
     '# dependencies = ["openai", "mcp==2.0.0", "httpx", "httpx2", "tenacity"]\n'
     "# ///\n"
 )
-CHAT_PROGRAM_SOURCE = bundle_program(CHAT_PROGRAM, compaction, core)
-MCP_CHAT_PROGRAM_SOURCE = bundle_program(CHAT_PROGRAM, mcp, compaction, core)
+CHAT_PROGRAM_SOURCE = bundle_program(CHAT_PROGRAM, mcp, compaction, core)
 
 
 async def launch_chat_program(
@@ -45,7 +44,6 @@ async def launch_chat_program(
     system_prompt: str | None,
     prompt: str | Messages | None,
     *,
-    source_with_mcp: str | None = None,
     extra_args: Sequence[str] = (),
     env: dict[str, str] | None = None,
     activate: bool = True,
@@ -60,7 +58,6 @@ async def launch_chat_program(
     if system_prompt:
         args.append(f"--system-prompt={system_prompt}")
     if mcp_urls:
-        source = source_with_mcp or source
         args.append(
             "--mcp-config="
             + json.dumps(

verifiers-0.3.2.dev67/verifiers/v1/harnesses/utils/mcp.py

@@ -0,0 +1,201 @@
+import asyncio
+from collections.abc import AsyncIterator, Awaitable, Callable, Sequence
+from contextlib import AsyncExitStack, asynccontextmanager, suppress
+from typing import TYPE_CHECKING, Any, TypeVar, cast
+
+from anyio import CancelScope
+
+if TYPE_CHECKING:
+    from mcp import Client
+
+MCP_CALL_ATTEMPTS = 6
+MCP_TIMEOUT = 600.0
+
+T = TypeVar("T")
+
+
+@asynccontextmanager
+async def mcp_client(spec: dict[str, Any]) -> AsyncIterator["Client"]:
+    """Open one fresh MCP client in the caller's task.
+
+    The client negotiates the newest protocol and falls back for older servers.
+    Teardown failures after the body completes are suppressed so closing noise cannot
+    fail or replay a call whose result is already available.
+    """
+    # Bundled chat programs also run without tools; load MCP only when it is used.
+    import httpx2
+    from mcp import Client
+    from mcp.client.streamable_http import (
+        create_mcp_http_client,
+        streamable_http_client,
+    )
+
+    stack = AsyncExitStack()
+    try:
+        http_client = await stack.enter_async_context(
+            create_mcp_http_client(
+                headers=spec.get("headers") or None,
+                timeout=httpx2.Timeout(
+                    spec.get("timeout", MCP_TIMEOUT),
+                    connect=spec.get("connect_timeout", 5.0),
+                ),
+            )
+        )
+        transport = streamable_http_client(spec["url"], http_client=http_client)
+        yield await stack.enter_async_context(Client(transport))
+    finally:
+        with suppress(Exception):
+            await stack.aclose()
+
+
+class MCPConnection:
+    """One rollout-owned client, with operations serialized across reconnects.
+
+    The SDK's AnyIO contexts enter and exit in the owner task, even when discovery
+    runs under wait_for or a tool call is cancelled by another task.
+    """
+
+    def __init__(self, spec: dict[str, Any]):
+        # Bundled programs without MCP should not import the retry machinery.
+        from tenacity import AsyncRetrying, stop_after_attempt, wait_exponential_jitter
+
+        self.spec = spec
+        self.task: asyncio.Task[None] | None = None
+        self.ready: asyncio.Future[Client] | None = None
+        self.cancel_scope: CancelScope | None = None
+        self.lock = asyncio.Lock()
+        self.run = AsyncRetrying(
+            stop=stop_after_attempt(MCP_CALL_ATTEMPTS),
+            wait=wait_exponential_jitter(initial=0.5, max=30),
+            reraise=True,
+        ).wraps(self._run)
+
+    async def serve(
+        self, ready: asyncio.Future["Client"], cancel_scope: CancelScope
+    ) -> None:
+        stack = AsyncExitStack()
+        try:
+            stack.enter_context(cancel_scope)
+            client = await stack.enter_async_context(mcp_client(self.spec))
+            ready.set_result(client)
+            await asyncio.Future()
+        except asyncio.CancelledError:
+            pass
+        except Exception as error:  # noqa: BLE001 - deliver owner failures to the caller
+            if not ready.done():
+                ready.set_exception(error)
+        finally:
+            if not ready.done():
+                ready.cancel()
+            # Owner teardown must not replace the caller's error with cancellation.
+            with suppress(asyncio.CancelledError):
+                await stack.aclose()
+
+    async def _run(self, operation: Callable[["Client"], Awaitable[T]]) -> T:
+        await self.lock.acquire()
+        try:
+            if self.task is None or self.task.done():
+                self.ready = asyncio.get_running_loop().create_future()
+                self.cancel_scope = CancelScope()
+                self.task = asyncio.create_task(
+                    self.serve(self.ready, self.cancel_scope)
+                )
+            assert self.ready is not None
+            client = await self.ready
+            return await operation(client)
+        except BaseException as error:
+            await self.aclose(abort=isinstance(error, asyncio.CancelledError))
+            raise
+        finally:
+            self.lock.release()
+
+    async def aclose(self, *, abort: bool = False) -> None:
+        if self.task is None:
+            return
+        task, self.task = self.task, None
+        cancel_scope = self.cancel_scope
+        assert cancel_scope is not None
+        caller = asyncio.current_task()
+        assert caller is not None
+        # Stack cleanup can follow cancellation while awaiting a model or local tool.
+        if abort or caller.cancelling():
+            cancel_scope.cancel()
+        else:
+            task.cancel()
+        try:
+            await asyncio.shield(task)
+        except asyncio.CancelledError:
+            cancel_scope.cancel()
+            await asyncio.shield(task)
+            raise
+
+
+async def connect_mcp(
+    config: dict[str, Any], stack: AsyncExitStack, reserved: set[str] | None = None
+) -> tuple[
+    list[dict[str, Any]],
+    dict[str, tuple[str, str]],
+    dict[str, MCPConnection],
+]:
+    """Enumerate MCP tools and return their schemas, dispatch map, and servers."""
+    tool_schemas: list[dict[str, Any]] = []
+    dispatch: dict[str, tuple[str, str]] = {}
+    servers: dict[str, MCPConnection] = {}
+    reserved = reserved or set()
+    for name, spec in config.get("mcpServers", {}).items():
+        server = servers[name] = MCPConnection(spec)
+        stack.push_async_callback(server.aclose)
+        result = await server.run(lambda client: client.list_tools())
+        for tool in result.tools:
+            full = f"{name}_{tool.name}" if name else tool.name
+            if full in reserved or full in dispatch:
+                raise ValueError(
+                    f"duplicate tool name {full!r}; keep MCP tool names qualified"
+                )
+            tool_schemas.append(
+                {
+                    "type": "function",
+                    "function": {
+                        "name": full,
+                        "description": tool.description or "",
+                        "parameters": tool.input_schema,
+                    },
+                }
+            )
+            dispatch[full] = (name, tool.name)
+    return tool_schemas, dispatch, servers
+
+
+def mcp_content_to_chat_content(
+    blocks: Sequence[Any],
+) -> str | list[dict[str, Any]]:
+    """Convert MCP content blocks to OpenAI chat tool-result content."""
+    parts = []
+    for block in blocks:
+        if block.type == "text":
+            parts.append({"type": "text", "text": block.text})
+        elif block.type == "image":
+            url = f"data:{block.mime_type};base64,{block.data}"
+            parts.append({"type": "image_url", "image_url": {"url": url}})
+        else:
+            parts.append({"type": "text", "text": str(block)})
+    if not parts:
+        return str(blocks)
+    if all(part["type"] == "text" for part in parts):
+        return "\n".join(cast(str, part["text"]) for part in parts)
+    return parts
+
+
+async def call_mcp(
+    servers: dict[str, MCPConnection],
+    dispatch: dict[str, tuple[str, str]],
+    name: str,
+    arguments: dict[str, Any],
+) -> str | list[dict[str, Any]]:
+    """Reuse the rollout's client, reconnecting before retrying a failed call."""
+    server_name, raw = dispatch[name]
+
+    result = await servers[server_name].run(
+        lambda client: client.call_tool(raw, arguments)
+    )
+    return mcp_content_to_chat_content(result.content)

{verifiers-0.3.2.dev65 → verifiers-0.3.2.dev67}/verifiers/v1/tasksets/harbor/taskset.py

@@ -379,7 +379,13 @@ def task_resources(environment, multiplier: float) -> TaskResources:
         memory=environment.memory_mb / 1024 * multiplier
         if environment.memory_mb
         else None,
-        gpu=str(environment.gpus) if environment.gpus else None,
+        gpu=(
+            f"{environment.gpu_types[0]}:{environment.gpus}"
+            if environment.gpu_types
+            else str(environment.gpus)
+        )
+        if environment.gpus
+        else None,
         disk=environment.storage_mb / 1024 * multiplier
         if environment.storage_mb
         else None,
@@ -694,7 +700,7 @@ def parse_verifier_environment(
         )
     unsupported = [
         field
-        for field in ("mcp_servers", "skills_dir", "gpu_types", "tpu")
+        for field in ("mcp_servers", "skills_dir", "tpu")
         if getattr(environment, field, None)
     ]
     if environment.os != TaskOS.LINUX or unsupported:

verifiers-0.3.2.dev65/verifiers/v1/harnesses/utils/mcp.py

@@ -1,127 +0,0 @@
-from collections.abc import AsyncIterator, Awaitable, Callable, Sequence
-from contextlib import AsyncExitStack, asynccontextmanager, suppress
-from typing import Any, TypeVar, cast
-
-import httpx2
-from mcp import Client
-from mcp.client.streamable_http import create_mcp_http_client, streamable_http_client
-from tenacity import AsyncRetrying, stop_after_attempt, wait_exponential_jitter
-
-MCP_CALL_ATTEMPTS = 6
-MCP_TIMEOUT = 600.0
-
-T = TypeVar("T")
-
-
-@asynccontextmanager
-async def mcp_client(spec: dict[str, Any]) -> AsyncIterator[Client]:
-    """Open one fresh MCP client in the caller's task.
-
-    The client negotiates the newest protocol and falls back for older servers.
-    Teardown failures after the body completes are suppressed so closing noise cannot
-    fail or replay a call whose result is already available.
-    """
-    stack = AsyncExitStack()
-    try:
-        http_client = await stack.enter_async_context(
-            create_mcp_http_client(
-                headers=spec.get("headers") or None,
-                timeout=httpx2.Timeout(
-                    spec.get("timeout", MCP_TIMEOUT),
-                    connect=spec.get("connect_timeout", 5.0),
-                ),
-            )
-        )
-        transport = streamable_http_client(spec["url"], http_client=http_client)
-        yield await stack.enter_async_context(Client(transport))
-    finally:
-        with suppress(Exception):
-            await stack.aclose()
-
-
-async def with_retry(call: Callable[[], Awaitable[T]]) -> T:
-    """Run one client operation with the existing at-least-once retries."""
-    async for attempt in AsyncRetrying(
-        stop=stop_after_attempt(MCP_CALL_ATTEMPTS),
-        wait=wait_exponential_jitter(initial=0.5, max=30),
-        reraise=True,
-    ):
-        with attempt:
-            return await call()
-    raise RuntimeError("retrying stopped without returning or raising")
-
-
-async def connect_mcp(
-    config: dict[str, Any], reserved: set[str] | None = None
-) -> tuple[
-    list[dict[str, Any]],
-    dict[str, tuple[str, str]],
-    dict[str, dict[str, Any]],
-]:
-    """Enumerate MCP tools and return their schemas, dispatch map, and servers."""
-    tool_schemas: list[dict[str, Any]] = []
-    dispatch: dict[str, tuple[str, str]] = {}
-    servers: dict[str, dict[str, Any]] = {}
-    reserved = reserved or set()
-    for name, spec in config.get("mcpServers", {}).items():
-        servers[name] = spec
-
-        async def list_tools(spec: dict[str, Any] = spec):
-            async with mcp_client(spec) as client:
-                return (await client.list_tools()).tools
-
-        for tool in await with_retry(list_tools):
-            full = f"{name}_{tool.name}" if name else tool.name
-            if full in reserved or full in dispatch:
-                raise ValueError(
-                    f"duplicate tool name {full!r}; keep MCP tool names qualified"
-                )
-            tool_schemas.append(
-                {
-                    "type": "function",
-                    "function": {
-                        "name": full,
-                        "description": tool.description or "",
-                        "parameters": tool.input_schema,
-                    },
-                }
-            )
-            dispatch[full] = (name, tool.name)
-    return tool_schemas, dispatch, servers
-
-
-def mcp_content_to_chat_content(
-    blocks: Sequence[Any],
-) -> str | list[dict[str, Any]]:
-    """Convert MCP content blocks to OpenAI chat tool-result content."""
-    parts = []
-    for block in blocks:
-        if block.type == "text":
-            parts.append({"type": "text", "text": block.text})
-        elif block.type == "image":
-            url = f"data:{block.mime_type};base64,{block.data}"
-            parts.append({"type": "image_url", "image_url": {"url": url}})
-        else:
-            parts.append({"type": "text", "text": str(block)})
-    if not parts:
-        return str(blocks)
-    if all(part["type"] == "text" for part in parts):
-        return "\n".join(cast(str, part["text"]) for part in parts)
-    return parts
-
-
-async def call_mcp(
-    servers: dict[str, dict[str, Any]],
-    dispatch: dict[str, tuple[str, str]],
-    name: str,
-    arguments: dict[str, Any],
-) -> str | list[dict[str, Any]]:
-    """Call one MCP tool with a fresh client per retry attempt."""
-    server_name, raw = dispatch[name]
-
-    async def call():
-        async with mcp_client(servers[server_name]) as client:
-            return await client.call_tool(raw, arguments)
-
-    result = await with_retry(call)
-    return mcp_content_to_chat_content(result.content)