verifiers 0.2.1.dev61__tar.gz → 0.2.1.dev63__tar.gz

{verifiers-0.2.1.dev61 → verifiers-0.2.1.dev63}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.1.dev61
+Version: 0.2.1.dev63
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.2.1.dev61 → verifiers-0.2.1.dev63}/verifiers/v1/harnesses/bash/program.py

@@ -1,6 +1,6 @@
 # /// script
 # requires-python = ">=3.10"
-# dependencies = ["openai", "mcp", "httpx"]
+# dependencies = ["openai", "mcp", "httpx", "tenacity"]
 # ///
 """Secrets arrive through argv so local tool subprocesses do not inherit them."""
 
@@ -8,14 +8,19 @@ import argparse
 import asyncio
 import json
 import subprocess
-from contextlib import AsyncExitStack
+from contextlib import AsyncExitStack, asynccontextmanager, suppress
 from pathlib import Path
 
 import httpx
 from openai import AsyncOpenAI
+from tenacity import AsyncRetrying, stop_after_attempt, wait_exponential_jitter
 
 SERPER_URL = "https://google.serper.dev/search"
 
+MCP_CALL_ATTEMPTS = 6
+MCP_TIMEOUT = httpx.Timeout(600.0, connect=5.0)  # the OpenAI SDK client defaults
+
+
 BASH_TOOL = {
     "type": "function",
     "function": {
@@ -177,29 +182,66 @@ async def chat(
     return completion.choices[0].message
 
 
-async def connect_mcp(
-    stack: AsyncExitStack, config: dict, reserved: set[str]
-) -> tuple[list[dict], dict]:
-    """Connect to each configured MCP server (a streamable-HTTP `url`); return
-    (tool schemas, dispatch mapping `<server>_<tool>` -> (session, raw tool name))."""
+@asynccontextmanager
+async def mcp_session(spec: dict):
+    """One fresh streamable-HTTP session to an MCP server, opened and closed within the caller's
+    task so AnyIO cancellation scopes stay correctly nested. A teardown failure after the body
+    completed is swallowed — the result is already in hand, and closing noise must not fail (or
+    replay) an already-answered call."""
     from mcp import ClientSession
     from mcp.client.streamable_http import (
         create_mcp_http_client,
         streamable_http_client,
     )
 
-    tool_schemas: list[dict] = []
-    dispatch: dict[str, tuple] = {}
-    for name, spec in config.get("mcpServers", {}).items():
+    stack = AsyncExitStack()
+    try:
         http_client = await stack.enter_async_context(
-            create_mcp_http_client(headers=spec.get("headers") or None)
+            create_mcp_http_client(
+                headers=spec.get("headers") or None, timeout=MCP_TIMEOUT
+            )
         )
         read, write, *_ = await stack.enter_async_context(
             streamable_http_client(spec["url"], http_client=http_client)
         )
         session = await stack.enter_async_context(ClientSession(read, write))
         await session.initialize()
-        for tool in (await session.list_tools()).tools:
+        yield session
+    finally:
+        with suppress(Exception):
+            await stack.aclose()
+
+
+async def with_retry(call):
+    """Run one session-scoped operation, retrying transient failures with backoff. A call whose
+    response was lost may be replayed — MCP has no idempotency key, so tools should tolerate
+    at-least-once delivery (a tool that fails reports through its result, not an exception)."""
+    async for attempt in AsyncRetrying(
+        stop=stop_after_attempt(MCP_CALL_ATTEMPTS),
+        wait=wait_exponential_jitter(initial=0.5, max=30),
+        reraise=True,
+    ):
+        with attempt:
+            return await call()
+
+
+async def connect_mcp(
+    config: dict, reserved: set[str]
+) -> tuple[list[dict], dict, dict]:
+    """Enumerate each configured MCP server's tools (a streamable-HTTP `url`); return (tool schemas,
+    dispatch mapping `<server>_<tool>` -> (server name, raw tool name), servers mapping name -> spec).
+    No session is held — a stateless-HTTP server is reconnected per call."""
+    tool_schemas: list[dict] = []
+    dispatch: dict[str, tuple] = {}
+    servers: dict[str, dict] = {}
+    for name, spec in config.get("mcpServers", {}).items():
+        servers[name] = spec
+
+        async def list_tools(spec: dict = spec):
+            async with mcp_session(spec) as session:
+                return (await session.list_tools()).tools
+
+        for tool in await with_retry(list_tools):
             # A server named "" (TOOL_PREFIX = None) advertises its tools bare.
             full = f"{name}_{tool.name}" if name else tool.name
             if full in reserved or full in dispatch:
@@ -216,8 +258,8 @@ async def connect_mcp(
                     },
                 }
             )
-            dispatch[full] = (session, tool.name)
-    return tool_schemas, dispatch
+            dispatch[full] = (name, tool.name)
+    return tool_schemas, dispatch, servers
 
 
 def mcp_content_to_chat_content(blocks) -> str | list[dict]:
@@ -237,9 +279,18 @@ def mcp_content_to_chat_content(blocks) -> str | list[dict]:
     return parts
 
 
-async def call_mcp(dispatch: dict, name: str, arguments: dict) -> str | list[dict]:
-    session, raw = dispatch[name]
-    result = await session.call_tool(raw, arguments)
+async def call_mcp(
+    servers: dict, dispatch: dict, name: str, arguments: dict
+) -> str | list[dict]:
+    """Call a tool on a fresh session per attempt — see `with_retry` for the replay semantics.
+    The result is converted outside the retry so a conversion failure fails once."""
+    server_name, raw = dispatch[name]
+
+    async def call():
+        async with mcp_session(servers[server_name]) as session:
+            return await session.call_tool(raw, arguments)
+
+    result = await with_retry(call)
     return mcp_content_to_chat_content(result.content)
 
 
@@ -268,84 +319,83 @@ async def main() -> None:
         initial = json.loads(payload)
     client = AsyncOpenAI(base_url=args.base_url, api_key=args.api_key)
     config = json.loads(args.mcp_config or "{}")
-    async with AsyncExitStack() as stack:
-        tools = [BASH_TOOL]
-        reserved = {"bash"}
-        if args.edit:
-            tools.append(EDIT_TOOL)
-            reserved.add("edit")
-        if args.search:
-            tools.append(SEARCH_TOOL)
-            reserved.add("search")
-        mcp_tools, dispatch = (
-            await connect_mcp(stack, config, reserved)
-            if config.get("mcpServers")
-            else ([], {})
-        )
-        tools += mcp_tools
-        messages = (
-            [{"role": "system", "content": args.system_prompt}]
-            if args.system_prompt
-            else []
-        )
-        if initial:
-            messages.extend(initial)
-        elif args.prompt:
-            messages.append({"role": "user", "content": args.prompt})
-        while True:
-            message = await chat(client, args.model, messages, tools)
-            messages.append(message.model_dump(exclude_none=True))
-            if not message.tool_calls:
-                break
-            for call in message.tool_calls:
-                name = call.function.name
-                try:
-                    tool_args = json.loads(call.function.arguments or "{}")
-                except json.JSONDecodeError as e:
-                    messages.append(
-                        {
-                            "role": "tool",
-                            "tool_call_id": call.id,
-                            "content": f"error: invalid JSON in tool arguments ({e}); resend the call with valid JSON",
-                        }
-                    )
-                    continue
-                # Valid JSON can still be a non-object (`[]`, `42`, `null`); the `.get(...)` calls
-                # below assume a dict, so reject anything else as a tool error rather than crashing.
-                if not isinstance(tool_args, dict):
-                    messages.append(
-                        {
-                            "role": "tool",
-                            "tool_call_id": call.id,
-                            "content": f"error: tool arguments must be a JSON object, got {type(tool_args).__name__}; resend as an object",
-                        }
-                    )
-                    continue
-                if name in dispatch:
-                    content = await call_mcp(dispatch, name, tool_args)
-                elif name == "bash":
-                    content = await asyncio.to_thread(
-                        run_bash, tool_args.get("command", "")
-                    )
-                elif name == "edit" and args.edit:
-                    content = await asyncio.to_thread(
-                        run_edit,
-                        tool_args.get("path"),
-                        tool_args.get("old_str"),
-                        tool_args.get("new_str"),
-                    )
-                elif name == "search" and args.search:
-                    content = await asyncio.to_thread(
-                        run_search,
-                        tool_args.get("query", ""),
-                        args.serper_key,
-                        tool_args.get("num_results", 5),
-                    )
-                else:
-                    content = f"error: unknown tool {name!r}"
+    tools = [BASH_TOOL]
+    reserved = {"bash"}
+    if args.edit:
+        tools.append(EDIT_TOOL)
+        reserved.add("edit")
+    if args.search:
+        tools.append(SEARCH_TOOL)
+        reserved.add("search")
+    mcp_tools, dispatch, servers = (
+        await connect_mcp(config, reserved)
+        if config.get("mcpServers")
+        else ([], {}, {})
+    )
+    tools += mcp_tools
+    messages = (
+        [{"role": "system", "content": args.system_prompt}]
+        if args.system_prompt
+        else []
+    )
+    if initial:
+        messages.extend(initial)
+    elif args.prompt:
+        messages.append({"role": "user", "content": args.prompt})
+    while True:
+        message = await chat(client, args.model, messages, tools)
+        messages.append(message.model_dump(exclude_none=True))
+        if not message.tool_calls:
+            break
+        for call in message.tool_calls:
+            name = call.function.name
+            try:
+                tool_args = json.loads(call.function.arguments or "{}")
+            except json.JSONDecodeError as e:
+                messages.append(
+                    {
+                        "role": "tool",
+                        "tool_call_id": call.id,
+                        "content": f"error: invalid JSON in tool arguments ({e}); resend the call with valid JSON",
+                    }
+                )
+                continue
+            # Valid JSON can still be a non-object (`[]`, `42`, `null`); the `.get(...)` calls
+            # below assume a dict, so reject anything else as a tool error rather than crashing.
+            if not isinstance(tool_args, dict):
                 messages.append(
-                    {"role": "tool", "tool_call_id": call.id, "content": content}
+                    {
+                        "role": "tool",
+                        "tool_call_id": call.id,
+                        "content": f"error: tool arguments must be a JSON object, got {type(tool_args).__name__}; resend as an object",
+                    }
+                )
+                continue
+            if name in dispatch:
+                content = await call_mcp(servers, dispatch, name, tool_args)
+            elif name == "bash":
+                content = await asyncio.to_thread(
+                    run_bash, tool_args.get("command", "")
                 )
+            elif name == "edit" and args.edit:
+                content = await asyncio.to_thread(
+                    run_edit,
+                    tool_args.get("path"),
+                    tool_args.get("old_str"),
+                    tool_args.get("new_str"),
+                )
+            elif name == "search" and args.search:
+                content = await asyncio.to_thread(
+                    run_search,
+                    tool_args.get("query", ""),
+                    args.serper_key,
+                    tool_args.get("num_results", 5),
+                )
+            else:
+                content = f"error: unknown tool {name!r}"
+            messages.append(
+                {"role": "tool", "tool_call_id": call.id, "content": content}
+            )
 
 
 if __name__ == "__main__":

{verifiers-0.2.1.dev61 → verifiers-0.2.1.dev63}/verifiers/v1/harnesses/claude_code/harness.py

@@ -21,7 +21,7 @@ curl -fsSL https://claude.ai/install.sh | HOME={home} bash -s {version}
 
 
 class ClaudeCodeHarnessConfig(HarnessConfig):
-    version: str = Field(default="2.1.207", pattern=r"^[A-Za-z0-9._+-]+$")
+    version: str = Field(default="2.1.214", pattern=r"^[A-Za-z0-9._+-]+$")
     """Claude Code release to install; pinned for reproducibility."""
 
 

{verifiers-0.2.1.dev61 → verifiers-0.2.1.dev63}/verifiers/v1/harnesses/codex/harness.py

@@ -36,7 +36,7 @@ chmod +x {bin}
 
 
 class CodexHarnessConfig(HarnessConfig):
-    version: str = "0.137.0"
+    version: str = "0.144.5"
     """Codex release to install (the `rust-v<version>` GitHub release); pinned for reproducibility."""
     multi_agent: bool = False
     """Enable Codex's native multi-agent v2 tools."""

{verifiers-0.2.1.dev61 → verifiers-0.2.1.dev63}/verifiers/v1/harnesses/kimi_code/harness.py

@@ -32,7 +32,7 @@ env \
 
 
 class KimiCodeHarnessConfig(HarnessConfig):
-    version: str = "0.14.3"
+    version: str = "0.27.0"
     """Kimi Code release to install, pinned for reproducibility."""
 
 

{verifiers-0.2.1.dev61 → verifiers-0.2.1.dev63}/verifiers/v1/harnesses/mini_swe_agent/harness.py

@@ -9,7 +9,7 @@ PROGRAM_SOURCE = (Path(__file__).resolve().parent / "program.py").read_text()
 
 
 class MiniSWEAgentHarnessConfig(HarnessConfig):
-    version: str = "2.2.8"
+    version: str = "2.4.5"
     """mini-swe-agent release to install, pinned for reproducibility."""
 
 

{verifiers-0.2.1.dev61 → verifiers-0.2.1.dev63}/verifiers/v1/harnesses/mini_swe_agent/program.py

@@ -1,6 +1,6 @@
 # /// script
 # requires-python = ">=3.10"
-# dependencies = ["mini-swe-agent=={version}"]
+# dependencies = ["mini-swe-agent=={version}", "litellm[proxy]"]
 # ///
 
 from minisweagent.run.mini import app

verifiers-0.2.1.dev63/verifiers/v1/harnesses/null/program.py

@@ -0,0 +1,217 @@
+# /// script
+# requires-python = ">=3.11"
+# dependencies = ["openai", "mcp", "httpx", "tenacity"]
+# ///
+"""The interception endpoint and secret arrive through argv rather than the environment."""
+
+import argparse
+import asyncio
+import json
+from contextlib import AsyncExitStack, asynccontextmanager, suppress
+from pathlib import Path
+
+import httpx
+from openai import AsyncOpenAI
+from tenacity import AsyncRetrying, stop_after_attempt, wait_exponential_jitter
+
+MCP_CALL_ATTEMPTS = 6
+MCP_TIMEOUT = httpx.Timeout(600.0, connect=5.0)  # the OpenAI SDK client defaults
+
+
+async def chat(
+    client: AsyncOpenAI, model: str, messages: list[dict], tools: list[dict]
+):
+    completion = await client.chat.completions.create(
+        model=model, messages=messages, tools=tools or None
+    )
+    return completion.choices[0].message
+
+
+@asynccontextmanager
+async def mcp_session(spec: dict):
+    """One fresh streamable-HTTP session to an MCP server, opened and closed within the caller's
+    task so AnyIO cancellation scopes stay correctly nested. A teardown failure after the body
+    completed is swallowed — the result is already in hand, and closing noise must not fail (or
+    replay) an already-answered call."""
+    from mcp import ClientSession
+    from mcp.client.streamable_http import (
+        create_mcp_http_client,
+        streamable_http_client,
+    )
+
+    stack = AsyncExitStack()
+    try:
+        http_client = await stack.enter_async_context(
+            create_mcp_http_client(
+                headers=spec.get("headers") or None, timeout=MCP_TIMEOUT
+            )
+        )
+        read, write, *_ = await stack.enter_async_context(
+            streamable_http_client(spec["url"], http_client=http_client)
+        )
+        session = await stack.enter_async_context(ClientSession(read, write))
+        await session.initialize()
+        yield session
+    finally:
+        with suppress(Exception):
+            await stack.aclose()
+
+
+async def with_retry(call):
+    """Run one session-scoped operation, retrying transient failures with backoff. A call whose
+    response was lost may be replayed — MCP has no idempotency key, so tools should tolerate
+    at-least-once delivery (a tool that fails reports through its result, not an exception)."""
+    async for attempt in AsyncRetrying(
+        stop=stop_after_attempt(MCP_CALL_ATTEMPTS),
+        wait=wait_exponential_jitter(initial=0.5, max=30),
+        reraise=True,
+    ):
+        with attempt:
+            return await call()
+
+
+async def connect_mcp(config: dict) -> tuple[list[dict], dict, dict]:
+    """Enumerate each configured MCP server's tools (a streamable-HTTP `url`); return (tool schemas,
+    dispatch mapping advertised name -> (server name, raw tool name), servers mapping name -> spec).
+    No session is held — a stateless-HTTP server is reconnected per call. Tools are advertised as
+    `<server>_<tool>`; a server named `""` (TOOL_PREFIX = None) advertises its tools bare, so names
+    must be unique across the rollout's servers."""
+    tool_schemas: list[dict] = []
+    dispatch: dict[str, tuple] = {}
+    servers: dict[str, dict] = {}
+    for name, spec in config.get("mcpServers", {}).items():
+        servers[name] = spec
+
+        async def list_tools(spec: dict = spec):
+            async with mcp_session(spec) as session:
+                return (await session.list_tools()).tools
+
+        for tool in await with_retry(list_tools):
+            full = f"{name}_{tool.name}" if name else tool.name
+            if full in dispatch:
+                raise ValueError(
+                    f"duplicate tool name {full!r} across servers; keep qualified names"
+                )
+            tool_schemas.append(
+                {
+                    "type": "function",
+                    "function": {
+                        "name": full,
+                        "description": tool.description or "",
+                        "parameters": tool.inputSchema,
+                    },
+                }
+            )
+            dispatch[full] = (name, tool.name)
+    return tool_schemas, dispatch, servers
+
+
+def mcp_content_to_chat_content(blocks) -> str | list[dict]:
+    parts = []
+    for block in blocks:
+        if block.type == "text":
+            parts.append({"type": "text", "text": block.text})
+        elif block.type == "image":
+            url = f"data:{block.mimeType};base64,{block.data}"
+            parts.append({"type": "image_url", "image_url": {"url": url}})
+        else:
+            parts.append({"type": "text", "text": str(block)})
+    if not parts:
+        return str(blocks)
+    if all(part["type"] == "text" for part in parts):
+        return "\n".join(part["text"] for part in parts)
+    return parts
+
+
+async def call_mcp(
+    servers: dict, dispatch: dict, name: str, arguments: dict
+) -> str | list[dict]:
+    """Call a tool on a fresh session per attempt — see `with_retry` for the replay semantics.
+    The result is converted outside the retry so a conversion failure fails once."""
+    server_name, raw = dispatch[name]
+
+    async def call():
+        async with mcp_session(servers[server_name]) as session:
+            return await session.call_tool(raw, arguments)
+
+    result = await with_retry(call)
+    return mcp_content_to_chat_content(result.content)
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--base-url", required=True)
+    parser.add_argument("--api-key", required=True)
+    parser.add_argument("--model", required=True)
+    parser.add_argument("--system-prompt", default="")
+    parser.add_argument("--prompt", default="")
+    parser.add_argument("--initial-messages-file", default="")
+    parser.add_argument("--mcp-config", default="")
+    return parser.parse_args()
+
+
+async def main() -> None:
+    args = parse_args()
+    initial = []
+    if args.initial_messages_file:
+        path = Path(args.initial_messages_file)
+        payload = path.read_bytes()
+        path.unlink()
+        initial = json.loads(payload)
+    client = AsyncOpenAI(base_url=args.base_url, api_key=args.api_key)
+    config = json.loads(args.mcp_config or "{}")
+    if config.get("mcpServers"):
+        # Bound only tool enumeration; each session is opened and closed within this task.
+        async with asyncio.timeout(60):
+            tools, dispatch, servers = await connect_mcp(config)
+    else:
+        tools, dispatch, servers = [], {}, {}
+    messages = (
+        [{"role": "system", "content": args.system_prompt}]
+        if args.system_prompt
+        else []
+    )
+    if initial:
+        messages.extend(initial)
+    elif args.prompt:
+        messages.append({"role": "user", "content": args.prompt})
+    while True:
+        message = await chat(client, args.model, messages, tools)
+        messages.append(message.model_dump(exclude_none=True))
+        if not message.tool_calls:
+            break
+        for call in message.tool_calls:
+            name = call.function.name
+            try:
+                tool_args = json.loads(call.function.arguments or "{}")
+            except json.JSONDecodeError as e:
+                messages.append(
+                    {
+                        "role": "tool",
+                        "tool_call_id": call.id,
+                        "content": f"error: invalid JSON in tool arguments ({e}); resend the call with valid JSON",
+                    }
+                )
+                continue
+            # Valid JSON can still be a non-object (`[]`, `42`, `null`); the MCP dispatch
+            # assumes a dict, so reject anything else as a tool error rather than crashing.
+            if not isinstance(tool_args, dict):
+                messages.append(
+                    {
+                        "role": "tool",
+                        "tool_call_id": call.id,
+                        "content": f"error: tool arguments must be a JSON object, got {type(tool_args).__name__}; resend as an object",
+                    }
+                )
+                continue
+            if name in dispatch:
+                content = await call_mcp(servers, dispatch, name, tool_args)
+            else:
+                content = f"error: unknown tool {name!r}"
+            messages.append(
+                {"role": "tool", "tool_call_id": call.id, "content": content}
+            )
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

{verifiers-0.2.1.dev61 → verifiers-0.2.1.dev63}/verifiers/v1/harnesses/pi/harness.py

@@ -87,7 +87,7 @@ export default async function isolatedMcp(pi) {{
 
 
 class PiHarnessConfig(HarnessConfig):
-    version: str = "0.80.6"
+    version: str = "0.80.10"
     """Pi release to install, pinned for reproducibility."""
 
 

{verifiers-0.2.1.dev61 → verifiers-0.2.1.dev63}/verifiers/v1/interception/server.py

@@ -346,10 +346,10 @@ class InterceptionServer(Interception):
         if (
             session.user is not None
             and session.trace.task.data.prompt is None
-            and session.trace.num_turns == 0
+            and all(m.role != "assistant" for m in prompt)
         ):
             if session.opening is None:
-                session.opening = await session.user("")
+                session.opening = await session.user("", len(prompt))
             body = dialect.extend(body, None, session.opening)
             prompt = [*prompt, *session.opening]
             # If the simulator ended at the open (its task's `@stop` now fires), the loop's
@@ -498,8 +498,11 @@ class InterceptionServer(Interception):
                 # when there's no user simulator to keep the conversation going.
                 if response.message.tool_calls or session.user is None:
                     return serve(response)
+                prompt = [*prompt, response.message]
                 try:
-                    user_messages = await session.user(response.message.content or "")
+                    user_messages = await session.user(
+                        response.message.content or "", len(prompt)
+                    )
                 except RolloutError as e:
                     return self._fail(session, dialect, e)
                 except Exception as e:
@@ -515,7 +518,7 @@ class InterceptionServer(Interception):
                 # `self.state`), caught by `refused()` at the top of the next iteration — the
                 # interception server holds no opinion about the state's contents.
                 body = dialect.extend(body, response.raw, user_messages)
-                prompt = [*prompt, response.message, *user_messages]
+                prompt = [*prompt, *user_messages]
                 # The simulator changed the payload, so this is a new operation not a retry.
                 headers.popall("idempotency-key", None)
                 headers.popall("x-idempotency-key", None)

{verifiers-0.2.1.dev61 → verifiers-0.2.1.dev63}/verifiers/v1/mcp/__init__.py

@@ -2,12 +2,12 @@
 
 from verifiers.v1.mcp.launch import (
     Respond,
-    connect_user,
     serve,
     SharedToolServer,
     serve_shared,
     serve_tools,
     serve_user,
+    user_respond,
 )
 from verifiers.v1.mcp.server import ServerBase
 from verifiers.v1.mcp.toolset import SharedToolsetConfig, Toolset, ToolsetConfig
@@ -21,10 +21,10 @@ __all__ = [
     "User",
     "UserConfig",
     "Respond",
-    "connect_user",
     "serve",
     "SharedToolServer",
     "serve_shared",
     "serve_tools",
     "serve_user",
+    "user_respond",
 ]