verifiers 0.2.1.dev62__tar.gz → 0.2.1.dev66__tar.gz

{verifiers-0.2.1.dev62 → verifiers-0.2.1.dev66}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.1.dev62
+Version: 0.2.1.dev66
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers
@@ -24,33 +24,32 @@ Requires-Python: <3.14,>=3.11
 Requires-Dist: aiohttp>=3.9.0
 Requires-Dist: aiolimiter>=1.2.1
 Requires-Dist: anthropic>=0.78.0
-Requires-Dist: datasets<4.7.0,>=3.0.0
-Requires-Dist: gepa
+Requires-Dist: datasets<6.0.0,>=3.3.0
+Requires-Dist: gepa>=0.0.6
 Requires-Dist: httpx>=0.27.0
 Requires-Dist: loguru>=0.7.0
 Requires-Dist: math-verify>=0.8.0
-Requires-Dist: mcp>=1.14.1
+Requires-Dist: mcp<2,>=1.24.0
 Requires-Dist: msgpack>=1.1.2
-Requires-Dist: numpy
-Requires-Dist: openai-agents>=0.0.7
-Requires-Dist: openai>=1.108.1
-Requires-Dist: prime-pydantic-config[toml]>=0.3.0.dev86
+Requires-Dist: numpy>=2.1.0
+Requires-Dist: openai-agents>=0.8.2
+Requires-Dist: openai>=2.9.0
+Requires-Dist: prime-pydantic-config[toml]>=0.4.2
 Requires-Dist: prime-sandboxes>=0.2.31
 Requires-Dist: prime-tunnel>=0.1.8
-Requires-Dist: pydantic>=2.11.9
+Requires-Dist: pydantic>=2.12.3
 Requires-Dist: pyzmq>=27.1.0
 Requires-Dist: renderers>=0.1.8
 Requires-Dist: requests
-Requires-Dist: rich
+Requires-Dist: rich>=11.0.0
 Requires-Dist: setproctitle>=1.3.0
 Requires-Dist: tenacity>=8.5.0
 Requires-Dist: tomli-w>=1.0.0
-Requires-Dist: typing-extensions; python_version < '3.12'
+Requires-Dist: typing-extensions>=4.12.2
 Requires-Dist: uvloop>=0.21.0; sys_platform != 'win32' and sys_platform != 'cygwin' and platform_python_implementation != 'PyPy'
 Provides-Extra: browser
-Requires-Dist: aiohttp>=3.9.0; extra == 'browser'
 Requires-Dist: python-dotenv>=1.0.0; extra == 'browser'
-Requires-Dist: stagehand>=3.0.0; extra == 'browser'
+Requires-Dist: stagehand>=3.4.0; extra == 'browser'
 Provides-Extra: harbor
 Requires-Dist: harbor==0.14.0; (python_version >= '3.12') and extra == 'harbor'
 Provides-Extra: modal
@@ -59,23 +58,8 @@ Provides-Extra: notebook
 Requires-Dist: nest-asyncio>=1.6.0; extra == 'notebook'
 Provides-Extra: openenv
 Requires-Dist: openenv>=0.4.1; extra == 'openenv'
-Provides-Extra: quest
-Requires-Dist: certifi; extra == 'quest'
-Requires-Dist: pillow; extra == 'quest'
-Requires-Dist: pymupdf; extra == 'quest'
 Provides-Extra: rg
-Requires-Dist: reasoning-gym; extra == 'rg'
-Provides-Extra: rl
-Requires-Dist: accelerate>=1.4.0; extra == 'rl'
-Requires-Dist: deepspeed>=0.17.6; extra == 'rl'
-Requires-Dist: flash-attn>=2.8.3; extra == 'rl'
-Requires-Dist: liger-kernel>=0.5.10; extra == 'rl'
-Requires-Dist: peft; extra == 'rl'
-Requires-Dist: requests; extra == 'rl'
-Requires-Dist: torch<2.9.0,>=2.8.0; extra == 'rl'
-Requires-Dist: transformers>=4.56.2; extra == 'rl'
-Requires-Dist: vllm<0.11.0,>=0.10.0; extra == 'rl'
-Requires-Dist: wandb; extra == 'rl'
+Requires-Dist: reasoning-gym>=0.1.8; extra == 'rg'
 Provides-Extra: ta
 Requires-Dist: nltk>=3.9.2; extra == 'ta'
 Requires-Dist: textarena==0.7.4; extra == 'ta'

{verifiers-0.2.1.dev62 → verifiers-0.2.1.dev66}/pyproject.toml

@@ -28,47 +28,45 @@ classifiers = [
 
 dependencies = [
     "anthropic>=0.78.0",
-    "datasets>=3.0.0, <4.7.0",
-    "numpy",
+    "datasets>=3.3.0,<6.0.0",
+    "numpy>=2.1.0",
     "math-verify>=0.8.0",
-    "mcp>=1.14.1",
-    "openai>=1.108.1",
-    "openai-agents>=0.0.7",
+    "mcp>=1.24.0,<2",
+    "openai>=2.9.0",
+    "openai-agents>=0.8.2",
     "prime-tunnel>=0.1.8",
     "prime-sandboxes>=0.2.31",
-    "pydantic>=2.11.9",
+    "pydantic>=2.12.3",
     "requests",
-    "rich",
+    "rich>=11.0.0",
     "tenacity>=8.5.0",
-    "gepa",
+    "gepa>=0.0.6",
     "pyzmq>=27.1.0",
     "msgpack>=1.1.2",
     "aiolimiter>=1.2.1",
     "setproctitle>=1.3.0",
     "httpx>=0.27.0",
     "aiohttp>=3.9.0",
-    "prime-pydantic-config[toml]>=0.3.0.dev86",
+    "prime-pydantic-config[toml]>=0.4.2",
     "uvloop>=0.21.0; sys_platform != 'win32' and sys_platform != 'cygwin' and platform_python_implementation != 'PyPy'",
     "loguru>=0.7.0",
     "tomli-w>=1.0.0",
     "renderers>=0.1.8",
-    "typing_extensions; python_version < '3.12'",
+    "typing_extensions>=4.12.2",
 ]
 
 [dependency-groups]
 dev = [
-    "ruff",
-    "pre-commit",
+    "ruff>=0.9.0",
+    "pre-commit>=2.19.0",
     "ty>=0.0.1a29,<0.0.22",
-    "pytest>=7.0.0",
+    "pytest>=7.0.0,<9.1.0",
     "pytest-asyncio>=0.21.0",
     "pytest-cov>=4.0.0",
     "pytest-xdist>=3.8.0",
-    "ipykernel",
-    "ipywidgets",
-    "reasoning-gym",
-    "stagehand>=3.0.0",
-    "aiohttp>=3.9.0",
+    "nest-asyncio>=1.6.0",
+    "reasoning-gym>=0.1.8",
+    "stagehand>=3.4.0",
     "python-dotenv>=1.0.0",
     "nltk>=3.9.2",
     "textarena==0.7.4",
@@ -85,15 +83,14 @@ examples = [
 
 [project.optional-dependencies]
 rg = [
-    "reasoning-gym",
+    "reasoning-gym>=0.1.8",
 ]
 ta = [
     "textarena==0.7.4",
     "nltk>=3.9.2",
 ]
 browser = [
-    "stagehand>=3.0.0",
-    "aiohttp>=3.9.0",
+    "stagehand>=3.4.0",
     "python-dotenv>=1.0.0",
 ]
 harbor = [
@@ -109,24 +106,6 @@ notebook = [
 openenv = [
     "openenv>=0.4.1",
 ]
-quest = [
-    "certifi",
-    "pillow",
-    "pymupdf",
-]
-rl = [
-    "torch>=2.8.0,<2.9.0",
-    "transformers>=4.56.2",
-    "accelerate>=1.4.0",
-    "requests",
-    "peft",
-    "wandb",
-    "vllm>=0.10.0,<0.11.0",
-    "liger-kernel>=0.5.10",
-    "deepspeed>=0.17.6",
-    "flash-attn>=2.8.3",
-]
-
 [tool.uv]
 preview = true
 required-version = ">=0.11.1"
@@ -159,12 +138,6 @@ prime-sandboxes = false
 prime-pydantic-config = false
 renderers = false
 
-[tool.uv.extra-build-dependencies]
-flash-attn = [{ requirement = "torch", match-runtime = true }]
-
-[tool.uv.extra-build-variables]
-flash-attn = { FLASH_ATTENTION_SKIP_CUDA_BUILD = "TRUE" }
-
 [project.scripts]
 vf-eval = "verifiers.scripts.eval:main"
 eval = "verifiers.v1.cli.eval.main:main"

{verifiers-0.2.1.dev62 → verifiers-0.2.1.dev66}/verifiers/v1/harnesses/__init__.py

@@ -11,6 +11,7 @@ from verifiers.v1.harnesses.mini_swe_agent import (
 )
 from verifiers.v1.harnesses.null import NullHarness, NullHarnessConfig
 from verifiers.v1.harnesses.pi import PiHarness, PiHarnessConfig
+from verifiers.v1.harnesses.pool import PoolHarness, PoolHarnessConfig
 from verifiers.v1.harnesses.rlm import RLMHarness, RLMHarnessConfig
 from verifiers.v1.harnesses.terminus_2 import Terminus2Harness, Terminus2HarnessConfig
 
@@ -27,6 +28,8 @@ __all__ = [
     "NullHarnessConfig",
     "PiHarness",
     "PiHarnessConfig",
+    "PoolHarness",
+    "PoolHarnessConfig",
     "MiniSWEAgentHarness",
     "MiniSWEAgentHarnessConfig",
     "RLMHarness",

{verifiers-0.2.1.dev62 → verifiers-0.2.1.dev66}/verifiers/v1/harnesses/bash/program.py

@@ -1,6 +1,6 @@
 # /// script
 # requires-python = ">=3.10"
-# dependencies = ["openai", "mcp", "httpx"]
+# dependencies = ["openai", "mcp", "httpx", "tenacity"]
 # ///
 """Secrets arrive through argv so local tool subprocesses do not inherit them."""
 
@@ -8,14 +8,19 @@ import argparse
 import asyncio
 import json
 import subprocess
-from contextlib import AsyncExitStack
+from contextlib import AsyncExitStack, asynccontextmanager, suppress
 from pathlib import Path
 
 import httpx
 from openai import AsyncOpenAI
+from tenacity import AsyncRetrying, stop_after_attempt, wait_exponential_jitter
 
 SERPER_URL = "https://google.serper.dev/search"
 
+MCP_CALL_ATTEMPTS = 6
+MCP_TIMEOUT = httpx.Timeout(600.0, connect=5.0)  # the OpenAI SDK client defaults
+
+
 BASH_TOOL = {
     "type": "function",
     "function": {
@@ -177,29 +182,66 @@ async def chat(
     return completion.choices[0].message
 
 
-async def connect_mcp(
-    stack: AsyncExitStack, config: dict, reserved: set[str]
-) -> tuple[list[dict], dict]:
-    """Connect to each configured MCP server (a streamable-HTTP `url`); return
-    (tool schemas, dispatch mapping `<server>_<tool>` -> (session, raw tool name))."""
+@asynccontextmanager
+async def mcp_session(spec: dict):
+    """One fresh streamable-HTTP session to an MCP server, opened and closed within the caller's
+    task so AnyIO cancellation scopes stay correctly nested. A teardown failure after the body
+    completed is swallowed — the result is already in hand, and closing noise must not fail (or
+    replay) an already-answered call."""
     from mcp import ClientSession
     from mcp.client.streamable_http import (
         create_mcp_http_client,
         streamable_http_client,
     )
 
-    tool_schemas: list[dict] = []
-    dispatch: dict[str, tuple] = {}
-    for name, spec in config.get("mcpServers", {}).items():
+    stack = AsyncExitStack()
+    try:
         http_client = await stack.enter_async_context(
-            create_mcp_http_client(headers=spec.get("headers") or None)
+            create_mcp_http_client(
+                headers=spec.get("headers") or None, timeout=MCP_TIMEOUT
+            )
         )
         read, write, *_ = await stack.enter_async_context(
             streamable_http_client(spec["url"], http_client=http_client)
         )
         session = await stack.enter_async_context(ClientSession(read, write))
         await session.initialize()
-        for tool in (await session.list_tools()).tools:
+        yield session
+    finally:
+        with suppress(Exception):
+            await stack.aclose()
+
+
+async def with_retry(call):
+    """Run one session-scoped operation, retrying transient failures with backoff. A call whose
+    response was lost may be replayed — MCP has no idempotency key, so tools should tolerate
+    at-least-once delivery (a tool that fails reports through its result, not an exception)."""
+    async for attempt in AsyncRetrying(
+        stop=stop_after_attempt(MCP_CALL_ATTEMPTS),
+        wait=wait_exponential_jitter(initial=0.5, max=30),
+        reraise=True,
+    ):
+        with attempt:
+            return await call()
+
+
+async def connect_mcp(
+    config: dict, reserved: set[str]
+) -> tuple[list[dict], dict, dict]:
+    """Enumerate each configured MCP server's tools (a streamable-HTTP `url`); return (tool schemas,
+    dispatch mapping `<server>_<tool>` -> (server name, raw tool name), servers mapping name -> spec).
+    No session is held — a stateless-HTTP server is reconnected per call."""
+    tool_schemas: list[dict] = []
+    dispatch: dict[str, tuple] = {}
+    servers: dict[str, dict] = {}
+    for name, spec in config.get("mcpServers", {}).items():
+        servers[name] = spec
+
+        async def list_tools(spec: dict = spec):
+            async with mcp_session(spec) as session:
+                return (await session.list_tools()).tools
+
+        for tool in await with_retry(list_tools):
             # A server named "" (TOOL_PREFIX = None) advertises its tools bare.
             full = f"{name}_{tool.name}" if name else tool.name
             if full in reserved or full in dispatch:
@@ -216,8 +258,8 @@ async def connect_mcp(
                     },
                 }
             )
-            dispatch[full] = (session, tool.name)
-    return tool_schemas, dispatch
+            dispatch[full] = (name, tool.name)
+    return tool_schemas, dispatch, servers
 
 
 def mcp_content_to_chat_content(blocks) -> str | list[dict]:
@@ -237,9 +279,18 @@ def mcp_content_to_chat_content(blocks) -> str | list[dict]:
     return parts
 
 
-async def call_mcp(dispatch: dict, name: str, arguments: dict) -> str | list[dict]:
-    session, raw = dispatch[name]
-    result = await session.call_tool(raw, arguments)
+async def call_mcp(
+    servers: dict, dispatch: dict, name: str, arguments: dict
+) -> str | list[dict]:
+    """Call a tool on a fresh session per attempt — see `with_retry` for the replay semantics.
+    The result is converted outside the retry so a conversion failure fails once."""
+    server_name, raw = dispatch[name]
+
+    async def call():
+        async with mcp_session(servers[server_name]) as session:
+            return await session.call_tool(raw, arguments)
+
+    result = await with_retry(call)
     return mcp_content_to_chat_content(result.content)
 
 
@@ -268,84 +319,83 @@ async def main() -> None:
         initial = json.loads(payload)
     client = AsyncOpenAI(base_url=args.base_url, api_key=args.api_key)
     config = json.loads(args.mcp_config or "{}")
-    async with AsyncExitStack() as stack:
-        tools = [BASH_TOOL]
-        reserved = {"bash"}
-        if args.edit:
-            tools.append(EDIT_TOOL)
-            reserved.add("edit")
-        if args.search:
-            tools.append(SEARCH_TOOL)
-            reserved.add("search")
-        mcp_tools, dispatch = (
-            await connect_mcp(stack, config, reserved)
-            if config.get("mcpServers")
-            else ([], {})
-        )
-        tools += mcp_tools
-        messages = (
-            [{"role": "system", "content": args.system_prompt}]
-            if args.system_prompt
-            else []
-        )
-        if initial:
-            messages.extend(initial)
-        elif args.prompt:
-            messages.append({"role": "user", "content": args.prompt})
-        while True:
-            message = await chat(client, args.model, messages, tools)
-            messages.append(message.model_dump(exclude_none=True))
-            if not message.tool_calls:
-                break
-            for call in message.tool_calls:
-                name = call.function.name
-                try:
-                    tool_args = json.loads(call.function.arguments or "{}")
-                except json.JSONDecodeError as e:
-                    messages.append(
-                        {
-                            "role": "tool",
-                            "tool_call_id": call.id,
-                            "content": f"error: invalid JSON in tool arguments ({e}); resend the call with valid JSON",
-                        }
-                    )
-                    continue
-                # Valid JSON can still be a non-object (`[]`, `42`, `null`); the `.get(...)` calls
-                # below assume a dict, so reject anything else as a tool error rather than crashing.
-                if not isinstance(tool_args, dict):
-                    messages.append(
-                        {
-                            "role": "tool",
-                            "tool_call_id": call.id,
-                            "content": f"error: tool arguments must be a JSON object, got {type(tool_args).__name__}; resend as an object",
-                        }
-                    )
-                    continue
-                if name in dispatch:
-                    content = await call_mcp(dispatch, name, tool_args)
-                elif name == "bash":
-                    content = await asyncio.to_thread(
-                        run_bash, tool_args.get("command", "")
-                    )
-                elif name == "edit" and args.edit:
-                    content = await asyncio.to_thread(
-                        run_edit,
-                        tool_args.get("path"),
-                        tool_args.get("old_str"),
-                        tool_args.get("new_str"),
-                    )
-                elif name == "search" and args.search:
-                    content = await asyncio.to_thread(
-                        run_search,
-                        tool_args.get("query", ""),
-                        args.serper_key,
-                        tool_args.get("num_results", 5),
-                    )
-                else:
-                    content = f"error: unknown tool {name!r}"
+    tools = [BASH_TOOL]
+    reserved = {"bash"}
+    if args.edit:
+        tools.append(EDIT_TOOL)
+        reserved.add("edit")
+    if args.search:
+        tools.append(SEARCH_TOOL)
+        reserved.add("search")
+    mcp_tools, dispatch, servers = (
+        await connect_mcp(config, reserved)
+        if config.get("mcpServers")
+        else ([], {}, {})
+    )
+    tools += mcp_tools
+    messages = (
+        [{"role": "system", "content": args.system_prompt}]
+        if args.system_prompt
+        else []
+    )
+    if initial:
+        messages.extend(initial)
+    elif args.prompt:
+        messages.append({"role": "user", "content": args.prompt})
+    while True:
+        message = await chat(client, args.model, messages, tools)
+        messages.append(message.model_dump(exclude_none=True))
+        if not message.tool_calls:
+            break
+        for call in message.tool_calls:
+            name = call.function.name
+            try:
+                tool_args = json.loads(call.function.arguments or "{}")
+            except json.JSONDecodeError as e:
+                messages.append(
+                    {
+                        "role": "tool",
+                        "tool_call_id": call.id,
+                        "content": f"error: invalid JSON in tool arguments ({e}); resend the call with valid JSON",
+                    }
+                )
+                continue
+            # Valid JSON can still be a non-object (`[]`, `42`, `null`); the `.get(...)` calls
+            # below assume a dict, so reject anything else as a tool error rather than crashing.
+            if not isinstance(tool_args, dict):
                 messages.append(
-                    {"role": "tool", "tool_call_id": call.id, "content": content}
+                    {
+                        "role": "tool",
+                        "tool_call_id": call.id,
+                        "content": f"error: tool arguments must be a JSON object, got {type(tool_args).__name__}; resend as an object",
+                    }
+                )
+                continue
+            if name in dispatch:
+                content = await call_mcp(servers, dispatch, name, tool_args)
+            elif name == "bash":
+                content = await asyncio.to_thread(
+                    run_bash, tool_args.get("command", "")
                 )
+            elif name == "edit" and args.edit:
+                content = await asyncio.to_thread(
+                    run_edit,
+                    tool_args.get("path"),
+                    tool_args.get("old_str"),
+                    tool_args.get("new_str"),
+                )
+            elif name == "search" and args.search:
+                content = await asyncio.to_thread(
+                    run_search,
+                    tool_args.get("query", ""),
+                    args.serper_key,
+                    tool_args.get("num_results", 5),
+                )
+            else:
+                content = f"error: unknown tool {name!r}"
+            messages.append(
+                {"role": "tool", "tool_call_id": call.id, "content": content}
+            )
 
 
 if __name__ == "__main__":