verifiers 0.2.1.dev41__tar.gz → 0.2.1.dev43__tar.gz

{verifiers-0.2.1.dev41 → verifiers-0.2.1.dev43}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.1.dev41
+Version: 0.2.1.dev43
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.2.1.dev41 → verifiers-0.2.1.dev43}/verifiers/v1/harnesses/default/program.py

@@ -198,7 +198,8 @@ async def connect_mcp(stack: AsyncExitStack, config: dict) -> tuple[list[dict],
         session = await stack.enter_async_context(ClientSession(read, write))
         await session.initialize()
         for tool in (await session.list_tools()).tools:
-            full = f"{name}_{tool.name}"
+            # A server named "" (TOOL_PREFIX = None) advertises its tools bare.
+            full = f"{name}_{tool.name}" if name else tool.name
             tool_schemas.append(
                 {
                     "type": "function",
@@ -259,12 +260,7 @@ async def main() -> None:
         payload = path.read_bytes()
         path.unlink()
         initial = json.loads(payload)
-    # No client-side retries: re-sending a request whose turn the interception already committed
-    # re-samples it and forks the trace into an extra dead-end branch. A generous timeout lets a
-    # slow turn finish instead of timing out and re-sending; genuine failures surface as error rows.
-    client = AsyncOpenAI(
-        base_url=args.base_url, api_key=args.api_key, timeout=1800.0, max_retries=0
-    )
+    client = AsyncOpenAI(base_url=args.base_url, api_key=args.api_key)
     config = json.loads(args.mcp_config or "{}")
     async with AsyncExitStack() as stack:
         mcp_tools, dispatch = (

{verifiers-0.2.1.dev41 → verifiers-0.2.1.dev43}/verifiers/v1/harnesses/null/program.py

@@ -24,7 +24,9 @@ async def chat(
 
 async def connect_mcp(stack: AsyncExitStack, config: dict) -> tuple[list[dict], dict]:
     """Connect to each configured MCP server (a streamable-HTTP `url`); return
-    (tool schemas, dispatch mapping `<server>_<tool>` -> (session, raw tool name))."""
+    (tool schemas, dispatch mapping advertised name -> (session, raw tool name)). Tools are
+    advertised as `<server>_<tool>`; a server named `""` (TOOL_PREFIX = None) advertises its
+    tools bare, so names must be unique across the rollout's servers."""
     from mcp import ClientSession
     from mcp.client.streamable_http import (
         create_mcp_http_client,
@@ -43,7 +45,11 @@ async def connect_mcp(stack: AsyncExitStack, config: dict) -> tuple[list[dict],
         session = await stack.enter_async_context(ClientSession(read, write))
         await session.initialize()
         for tool in (await session.list_tools()).tools:
-            full = f"{name}_{tool.name}"
+            full = f"{name}_{tool.name}" if name else tool.name
+            if full in dispatch:
+                raise ValueError(
+                    f"duplicate tool name {full!r} across servers; keep qualified names"
+                )
             tool_schemas.append(
                 {
                     "type": "function",

{verifiers-0.2.1.dev41 → verifiers-0.2.1.dev43}/verifiers/v1/interception/server.py

@@ -21,6 +21,7 @@ handled out-of-band (run by the harness).
 
 import asyncio
 import contextlib
+import hashlib
 import json
 import logging
 import secrets
@@ -49,7 +50,7 @@ from verifiers.v1.interception.tunnel import (
     make_tunnel,
 )
 from verifiers.v1.session import RolloutSession
-from verifiers.v1.types import Messages, Tool
+from verifiers.v1.types import Messages, Response, Tool
 
 logger = logging.getLogger(__name__)
 
@@ -61,6 +62,23 @@ logger = logging.getLogger(__name__)
 _MAX_REQUEST_BODY = 1024**3  # 1 GiB (aiohttp's default is 1 MiB)
 _KEEPALIVE_INTERVAL_SECONDS = 3
 _STREAM_QUEUE_MAXSIZE = 16
+# blake2b saturates ~1.7 GB/s, so a body up to this size hashes inline in well under a
+# millisecond; a larger one (bodies may reach `_MAX_REQUEST_BODY`) is hashed off the event
+# loop instead — see `_request_digest`.
+_HASH_INLINE_MAX = 1024**2  # 1 MiB
+
+
+def _body_digest(raw: bytes) -> bytes:
+    return hashlib.blake2b(raw, digest_size=16).digest()
+
+
+async def _request_digest(raw: bytes) -> bytes:
+    """Digest a request body for the retry-replay guard. Hash a small body inline; offload a
+    large one to a thread so it does not stall every multiplexed rollout on the event loop
+    (blake2b releases the GIL, so the thread runs the hash off the loop)."""
+    if len(raw) <= _HASH_INLINE_MAX:
+        return _body_digest(raw)
+    return await asyncio.to_thread(_body_digest, raw)
 
 
 def _completion_response(completion: dict | None) -> web.Response:
@@ -223,10 +241,38 @@ class InterceptionServer(Interception):
             body = from_json(raw)
         except ValueError:
             body = json.loads(raw)
+        req_hash = await _request_digest(raw)
         # Keep `read()` for aiohttp's size guard, then release its cache and our local
         # alias after parsing so the wire body does not survive model inference.
         request._read_bytes = None
         del raw
+        # Graph atomicity under retries. The harness SDK retries a transient failure by
+        # re-sending the byte-identical request; sampling it again would commit a second turn and
+        # fork the graph into a dead-end branch. Two cases, both resolved without re-sampling:
+        #   1. the first attempt already finished -> replay the recorded response;
+        #   2. the first attempt is still computing (a slow turn) -> await it and return its
+        #      result, so a slow turn is safe without an inflated client timeout.
+        # A growing conversation never repeats a body, so these only ever match a real retry; a
+        # failed attempt caches nothing and re-runs normally.
+        if session.last_request == req_hash and session.last_response is not None:
+            logger.debug("intercept replay: id=%s (retried request)", session.trace.id)
+            return _completion_response(session.last_response)
+
+        async def coalesced(inflight: "asyncio.Future[dict | None]") -> web.Response:
+            # Await the first attempt instead of re-sampling. None means it produced no servable
+            # response (it errored/refused), so let the SDK retry afresh.
+            logger.debug(
+                "intercept coalesce: id=%s (retry of in-flight turn)", session.trace.id
+            )
+            completion = await inflight
+            if completion is None:
+                return web.json_response(
+                    dialect.error_body("upstream attempt failed"), status=503
+                )
+            return _completion_response(completion)
+
+        if (inflight := session.inflight.get(req_hash)) is not None:
+            return await coalesced(inflight)
         logger.debug(
             "intercept %s: id=%s stream=%s",
             request.path,
@@ -256,112 +302,140 @@ class InterceptionServer(Interception):
         if dialect.streaming(body):
             return await self._stream(request, session, dialect, body, prompt, tools)
         headers = request.headers.copy()
+        # Claim the in-flight slot so a retry arriving mid-flight coalesces onto it (above) rather
+        # than starting a second inference. Re-check first: an identical request may have claimed
+        # it while we awaited the simulator opening. The get / create / assign below run with no
+        # await between them, so two concurrent identical requests can never both become owner.
+        if (inflight := session.inflight.get(req_hash)) is not None:
+            return await coalesced(inflight)
+        fut: asyncio.Future[dict | None] = asyncio.get_running_loop().create_future()
+        session.inflight[req_hash] = fut
+
+        def serve(response: Response) -> web.Response:
+            # Record the served turn and hand it to any coalesced retry, so a retried
+            # byte-identical request replays instead of re-sampling and forking the graph.
+            # `Response.raw` is the full native provider object (or the renderer's synthesized
+            # completion) that the server serializes back to the program.
+            session.last_request = req_hash
+            session.last_response = response.raw
+            if not fut.done():
+                fut.set_result(response.raw)
+            return _completion_response(response.raw)
+
         # A user simulator turns one program request into a multi-turn exchange: after each
         # model turn the simulator's reply is injected as a user turn and the model is
         # re-prompted, so a whole game plays out here and only the final assistant message
         # returns to the (simulator-unaware) program. Without a simulator the loop runs once.
-        completion: dict | None = (
-            None  # the latest turn's response, returned to the program
+        response: Response | None = (
+            None  # the latest committed turn (None until the first)
         )
-        while True:
-            try:
-                refused = await session.refused()
-            except RolloutError as e:
-                return self._fail(session, dialect, e)
-            except Exception as e:
-                return self._fail(
-                    session,
-                    dialect,
-                    TaskError(f"@stop failed: {type(e).__name__}: {e}"),
-                )
-            if refused is not None:
-                # Refuse the first model call to halt the harness; once a simulated
-                # conversation is under way, just end it and return the last turn cleanly.
-                if completion is None:
-                    return web.json_response(
-                        dialect.error_body(f"rollout stopped: {refused}"), status=400
+        try:
+            while True:
+                try:
+                    refused = await session.refused()
+                except RolloutError as e:
+                    return self._fail(session, dialect, e)
+                except Exception as e:
+                    return self._fail(
+                        session,
+                        dialect,
+                        TaskError(f"@stop failed: {type(e).__name__}: {e}"),
+                    )
+                if refused is not None:
+                    # Refuse the first model call to halt the harness; once a simulated
+                    # conversation is under way, just end it and return the last turn cleanly.
+                    if response is None:
+                        return web.json_response(
+                            dialect.error_body(f"rollout stopped: {refused}"),
+                            status=400,
+                        )
+                    return serve(response)
+                turn = graph.prepare_turn(session.trace, prompt)
+                session.error = None
+                try:
+                    response = await session.ctx.client.get_response(
+                        dialect,
+                        body,
+                        session.ctx.model,
+                        session.ctx.sampling,
+                        headers=headers,
+                        session_id=session.trace.id,
+                        turn=turn,
+                    )
+                except OverlongPromptError:
+                    # An overlong prompt is a budget limit, not a crash: end the rollout cleanly
+                    # as a truncation — return the last turn if there is one, else refuse to halt
+                    # the harness (same shape as `refused` above).
+                    session.trace.stop("context_length")
+                    logger.debug("prompt too long: id=%s", session.trace.id)
+                    if response is None:
+                        return web.json_response(
+                            dialect.error_body("rollout stopped: context_length"),
+                            status=400,
+                        )
+                    return serve(response)
+                except RolloutError as e:
+                    # Stash the real cause; the rollout re-raises it after the harness returns.
+                    # Relay the provider's status so the harness SDK retries 5xx/429 and not 4xx.
+                    session.error = e
+                    logger.warning(
+                        "model call failed: id=%s %s: %s",
+                        session.trace.id,
+                        type(e).__name__,
+                        e,
                     )
-                return _completion_response(completion)
-            turn = graph.prepare_turn(session.trace, prompt)
-            session.error = None
-            try:
-                response = await session.ctx.client.get_response(
-                    dialect,
-                    body,
-                    session.ctx.model,
-                    session.ctx.sampling,
-                    headers=headers,
-                    session_id=session.trace.id,
-                    turn=turn,
-                )
-            except OverlongPromptError:
-                # An overlong prompt is a budget limit, not a crash: end the rollout cleanly
-                # as a truncation — return the last turn if there is one, else refuse to halt
-                # the harness (same shape as `refused` above).
-                session.trace.stop("context_length")
-                logger.debug("prompt too long: id=%s", session.trace.id)
-                if completion is None:
                     return web.json_response(
-                        dialect.error_body("rollout stopped: context_length"),
-                        status=400,
+                        dialect.error_body(str(e)),
+                        status=getattr(e, "status_code", 502),
                     )
-                return _completion_response(completion)
-            except RolloutError as e:
-                # Stash the real cause; the rollout re-raises it after the harness returns. Relay
-                # the provider's status so the harness SDK retries 5xx/429 and not 4xx.
-                session.error = e
-                logger.warning(
-                    "model call failed: id=%s %s: %s",
-                    session.trace.id,
-                    type(e).__name__,
-                    e,
-                )
-                return web.json_response(
-                    dialect.error_body(str(e)), status=getattr(e, "status_code", 502)
-                )
-            except Exception as e:  # surface to the program as an API error
-                logger.warning(
-                    "model call failed: id=%s %s: %s",
+                except Exception as e:  # surface to the program as an API error
+                    logger.warning(
+                        "model call failed: id=%s %s: %s",
+                        session.trace.id,
+                        type(e).__name__,
+                        e,
+                    )
+                    return web.json_response(dialect.error_body(str(e)), status=502)
+                logger.debug(
+                    "intercept turn: id=%s tools=%d",
                     session.trace.id,
-                    type(e).__name__,
-                    e,
-                )
-                return web.json_response(dialect.error_body(str(e)), status=502)
-            # `Response.raw` is the full native provider object, or the renderer's synthesized
-            # completion object, that the server serializes for the program.
-            completion = response.raw
-            logger.debug(
-                "intercept turn: id=%s tools=%d",
-                session.trace.id,
-                len(response.message.tool_calls or []),
-            )
-            turn.commit(response, tools)  # one node per new message;
-            # branches fall out of walking the graph (see Trace.branches / verifiers.v1.graph)
-            # Hand back to the program when the model wants a tool (the program runs it) or
-            # when there's no user simulator to keep the conversation going.
-            if response.message.tool_calls or session.user is None:
-                return _completion_response(completion)
-            try:
-                user_messages = await session.user(response.message.content or "")
-            except RolloutError as e:
-                return self._fail(session, dialect, e)
-            except Exception as e:
-                return self._fail(
-                    session,
-                    dialect,
-                    UserError(f"user simulator failed: {type(e).__name__}: {e}"),
+                    len(response.message.tool_calls or []),
                 )
-            # Inject the model turn + the simulator's user turn(s): into the wire request for the
-            # next model call (`dialect.extend`, which keeps the model turn verbatim so reasoning
-            # survives) and into the typed prompt for the trace. The simulator ends the trajectory
-            # through its task's `@stop` (e.g. a `user_finished` flag it set on `self.state`),
-            # caught by `refused()` at the top of the next iteration — the interception server holds
-            # no opinion about the state's contents.
-            body = dialect.extend(body, completion, user_messages)
-            prompt = [*prompt, response.message, *user_messages]
-            # The simulator changed the payload, so this is a new operation rather than a retry.
-            headers.popall("idempotency-key", None)
-            headers.popall("x-idempotency-key", None)
+                turn.commit(response, tools)  # one node per new message;
+                # branches fall out of walking the graph (see Trace.branches / verifiers.v1.graph)
+                # Hand back to the program when the model wants a tool (the program runs it) or
+                # when there's no user simulator to keep the conversation going.
+                if response.message.tool_calls or session.user is None:
+                    return serve(response)
+                try:
+                    user_messages = await session.user(response.message.content or "")
+                except RolloutError as e:
+                    return self._fail(session, dialect, e)
+                except Exception as e:
+                    return self._fail(
+                        session,
+                        dialect,
+                        UserError(f"user simulator failed: {type(e).__name__}: {e}"),
+                    )
+                # Inject the model turn + the simulator's user turn(s): into the wire request for
+                # the next model call (`dialect.extend`, which keeps the model turn verbatim so
+                # reasoning survives) and into the typed prompt for the trace. The simulator ends
+                # the trajectory through its task's `@stop` (e.g. a `user_finished` flag it set on
+                # `self.state`), caught by `refused()` at the top of the next iteration — the
+                # interception server holds no opinion about the state's contents.
+                body = dialect.extend(body, response.raw, user_messages)
+                prompt = [*prompt, response.message, *user_messages]
+                # The simulator changed the payload, so this is a new operation not a retry.
+                headers.popall("idempotency-key", None)
+                headers.popall("x-idempotency-key", None)
+        finally:
+            # Free the in-flight slot and unblock any coalesced retry; None signals "no servable
+            # response" (an error/refuse return above), so the waiter surfaces a retryable error.
+            # Only clear our own entry — never one a later owner may have installed.
+            if session.inflight.get(req_hash) is fut:
+                session.inflight.pop(req_hash, None)
+            if not fut.done():
+                fut.set_result(None)
 
     async def _stream(
         self,

{verifiers-0.2.1.dev41 → verifiers-0.2.1.dev43}/verifiers/v1/mcp/server.py

@@ -113,8 +113,10 @@ def _import_ref(ref: str) -> object:
 
 
 class ServerBase(Generic[ConfigT, StateT]):
-    # The empty value falls back to the snake-cased class name.
-    TOOL_PREFIX: ClassVar[str] = ""
+    TOOL_PREFIX: ClassVar[str | None] = ""
+    """The empty value falls back to the snake-cased class name. None advertises the server's
+    tools bare (no `<server>_` prefix); name collisions across servers are then the taskset
+    author's concern."""
 
     EXTRAS: ClassVar[tuple[str, ...]] = ()
     """Package extras the server's module needs, applied at sandbox install."""
@@ -214,6 +216,8 @@ class ServerBase(Generic[ConfigT, StateT]):
 
     @property
     def server_name(self) -> str:
+        if self.TOOL_PREFIX is None:
+            return ""
         return self.TOOL_PREFIX or "".join(
             ("_" + c.lower() if c.isupper() else c) for c in type(self).__name__
         ).lstrip("_")

{verifiers-0.2.1.dev41 → verifiers-0.2.1.dev43}/verifiers/v1/session.py

@@ -8,6 +8,7 @@ injects the user simulator's replies, and stashes the real failure on `error`.
 turns.
 """
 
+import asyncio
 import logging
 from collections.abc import Awaitable, Callable
 from dataclasses import dataclass, field
@@ -81,6 +82,23 @@ class RolloutSession:
     (and may swallow it, or exit non-zero), so the rollout re-raises this original error once the
     harness returns — recording the real `ProviderError` instead of a secondary `HarnessError`.
     Reset before each model turn, so a successful retry clears it."""
+    last_request: bytes | None = None
+    """Digest of the most recently served request body; with `last_response`, the replay cache
+    that keeps the message graph atomic under harness-SDK retries. A retry re-sends the
+    byte-identical request; when it matches, the interception server replays the recorded
+    response instead of re-sampling and committing a second turn — which would fork the graph
+    into a dead-end branch. Only a fully served request is cached, so a genuinely failed attempt
+    still re-runs. Turns are issued sequentially (one outstanding request at a time), so a retry
+    is always of the most recent request — keeping only the last one is sufficient and bounded."""
+    last_response: dict | None = None
+    """The response returned for `last_request`, replayed verbatim on a retry."""
+    inflight: dict[bytes, "asyncio.Future[dict | None]"] = field(default_factory=dict)
+    """Body digest -> the future of the attempt currently computing it. A retry that arrives
+    while the first attempt is still in flight (a slow turn) awaits this future instead of
+    starting a second inference — the other half of retry atomicity (with `last_response`, which
+    covers a retry after the attempt finished). Because a slow turn is coalesced rather than
+    re-sampled, retries stay safe without an inflated client timeout. The future resolves to the
+    served response, or to None if the attempt produced no servable response (error/refuse)."""
 
     async def refused(self) -> str | None:
         """The framework's limits (turns / token budget) and `@stop` checks, run before each