verifiers 0.2.1.dev42__tar.gz → 0.2.1.dev44__tar.gz

{verifiers-0.2.1.dev42 → verifiers-0.2.1.dev44}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.1.dev42
+Version: 0.2.1.dev44
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.2.1.dev42 → verifiers-0.2.1.dev44}/verifiers/v1/cli/dashboard/eval.py

@@ -363,17 +363,14 @@ def _breakdown(done: list[Trace]) -> Table | None:
 
 def _tokens(trace: Trace) -> tuple[int, int, int | None, int | None, int]:
     """Input/output tokens summed across all branches: per branch, output is every assistant
-    (completion) token generated across its turns and input is its last turn's prompt — the full
-    final context the model saw. A rollout yields one training sample per branch (a linear trace
-    is a single branch; compaction and subagents add more), so the totals sum them — matching
-    `Trace.num_input_tokens` / `Trace.num_output_tokens`. (Output can exceed the final context —
-    reasoning tokens count toward completions but aren't re-fed — so it's not derived by
-    subtraction.)
-
-    Both counts read token ids when present and fall back to provider-reported usage when the
-    endpoint returns no token ids (e.g. plain OpenAI completions), so the counts aren't shown as
-    0/0. Returns the branch count from the same derived view so each dashboard tick materializes
-    it once."""
+    (completion) token generated across its turns and input is the fed-in tokens counted once
+    (system + user + tool) — the final sequence minus everything the model generated. A rollout
+    yields one training sample per branch (a linear trace is a single branch; compaction and
+    subagents add more), so the totals sum them — matching `Trace.num_input_tokens` /
+    `Trace.num_output_tokens`, whose sum is `num_total_tokens`.
+
+    Both counts come from provider-reported usage. Returns the branch count from the same derived
+    view so each dashboard tick materializes it once."""
     usage = trace.usage
     cached = usage.cached_input_tokens if usage else None
     reasoning = usage.reasoning_tokens if usage else None

{verifiers-0.2.1.dev42 → verifiers-0.2.1.dev44}/verifiers/v1/harnesses/default/program.py

@@ -260,12 +260,7 @@ async def main() -> None:
         payload = path.read_bytes()
         path.unlink()
         initial = json.loads(payload)
-    # No client-side retries: re-sending a request whose turn the interception already committed
-    # re-samples it and forks the trace into an extra dead-end branch. A generous timeout lets a
-    # slow turn finish instead of timing out and re-sending; genuine failures surface as error rows.
-    client = AsyncOpenAI(
-        base_url=args.base_url, api_key=args.api_key, timeout=1800.0, max_retries=0
-    )
+    client = AsyncOpenAI(base_url=args.base_url, api_key=args.api_key)
     config = json.loads(args.mcp_config or "{}")
     async with AsyncExitStack() as stack:
         mcp_tools, dispatch = (

{verifiers-0.2.1.dev42 → verifiers-0.2.1.dev44}/verifiers/v1/interception/server.py

@@ -21,6 +21,7 @@ handled out-of-band (run by the harness).
 
 import asyncio
 import contextlib
+import hashlib
 import json
 import logging
 import secrets
@@ -49,7 +50,7 @@ from verifiers.v1.interception.tunnel import (
     make_tunnel,
 )
 from verifiers.v1.session import RolloutSession
-from verifiers.v1.types import Messages, Tool
+from verifiers.v1.types import Messages, Response, Tool
 
 logger = logging.getLogger(__name__)
 
@@ -61,6 +62,23 @@ logger = logging.getLogger(__name__)
 _MAX_REQUEST_BODY = 1024**3  # 1 GiB (aiohttp's default is 1 MiB)
 _KEEPALIVE_INTERVAL_SECONDS = 3
 _STREAM_QUEUE_MAXSIZE = 16
+# blake2b saturates ~1.7 GB/s, so a body up to this size hashes inline in well under a
+# millisecond; a larger one (bodies may reach `_MAX_REQUEST_BODY`) is hashed off the event
+# loop instead — see `_request_digest`.
+_HASH_INLINE_MAX = 1024**2  # 1 MiB
+
+
+def _body_digest(raw: bytes) -> bytes:
+    return hashlib.blake2b(raw, digest_size=16).digest()
+
+
+async def _request_digest(raw: bytes) -> bytes:
+    """Digest a request body for the retry-replay guard. Hash a small body inline; offload a
+    large one to a thread so it does not stall every multiplexed rollout on the event loop
+    (blake2b releases the GIL, so the thread runs the hash off the loop)."""
+    if len(raw) <= _HASH_INLINE_MAX:
+        return _body_digest(raw)
+    return await asyncio.to_thread(_body_digest, raw)
 
 
 def _completion_response(completion: dict | None) -> web.Response:
@@ -223,10 +241,38 @@ class InterceptionServer(Interception):
             body = from_json(raw)
         except ValueError:
             body = json.loads(raw)
+        req_hash = await _request_digest(raw)
         # Keep `read()` for aiohttp's size guard, then release its cache and our local
         # alias after parsing so the wire body does not survive model inference.
         request._read_bytes = None
         del raw
+        # Graph atomicity under retries. The harness SDK retries a transient failure by
+        # re-sending the byte-identical request; sampling it again would commit a second turn and
+        # fork the graph into a dead-end branch. Two cases, both resolved without re-sampling:
+        #   1. the first attempt already finished -> replay the recorded response;
+        #   2. the first attempt is still computing (a slow turn) -> await it and return its
+        #      result, so a slow turn is safe without an inflated client timeout.
+        # A growing conversation never repeats a body, so these only ever match a real retry; a
+        # failed attempt caches nothing and re-runs normally.
+        if session.last_request == req_hash and session.last_response is not None:
+            logger.debug("intercept replay: id=%s (retried request)", session.trace.id)
+            return _completion_response(session.last_response)
+
+        async def coalesced(inflight: "asyncio.Future[dict | None]") -> web.Response:
+            # Await the first attempt instead of re-sampling. None means it produced no servable
+            # response (it errored/refused), so let the SDK retry afresh.
+            logger.debug(
+                "intercept coalesce: id=%s (retry of in-flight turn)", session.trace.id
+            )
+            completion = await inflight
+            if completion is None:
+                return web.json_response(
+                    dialect.error_body("upstream attempt failed"), status=503
+                )
+            return _completion_response(completion)
+
+        if (inflight := session.inflight.get(req_hash)) is not None:
+            return await coalesced(inflight)
         logger.debug(
             "intercept %s: id=%s stream=%s",
             request.path,
@@ -256,112 +302,140 @@ class InterceptionServer(Interception):
         if dialect.streaming(body):
             return await self._stream(request, session, dialect, body, prompt, tools)
         headers = request.headers.copy()
+        # Claim the in-flight slot so a retry arriving mid-flight coalesces onto it (above) rather
+        # than starting a second inference. Re-check first: an identical request may have claimed
+        # it while we awaited the simulator opening. The get / create / assign below run with no
+        # await between them, so two concurrent identical requests can never both become owner.
+        if (inflight := session.inflight.get(req_hash)) is not None:
+            return await coalesced(inflight)
+        fut: asyncio.Future[dict | None] = asyncio.get_running_loop().create_future()
+        session.inflight[req_hash] = fut
+
+        def serve(response: Response) -> web.Response:
+            # Record the served turn and hand it to any coalesced retry, so a retried
+            # byte-identical request replays instead of re-sampling and forking the graph.
+            # `Response.raw` is the full native provider object (or the renderer's synthesized
+            # completion) that the server serializes back to the program.
+            session.last_request = req_hash
+            session.last_response = response.raw
+            if not fut.done():
+                fut.set_result(response.raw)
+            return _completion_response(response.raw)
+
         # A user simulator turns one program request into a multi-turn exchange: after each
         # model turn the simulator's reply is injected as a user turn and the model is
         # re-prompted, so a whole game plays out here and only the final assistant message
         # returns to the (simulator-unaware) program. Without a simulator the loop runs once.
-        completion: dict | None = (
-            None  # the latest turn's response, returned to the program
+        response: Response | None = (
+            None  # the latest committed turn (None until the first)
         )
-        while True:
-            try:
-                refused = await session.refused()
-            except RolloutError as e:
-                return self._fail(session, dialect, e)
-            except Exception as e:
-                return self._fail(
-                    session,
-                    dialect,
-                    TaskError(f"@stop failed: {type(e).__name__}: {e}"),
-                )
-            if refused is not None:
-                # Refuse the first model call to halt the harness; once a simulated
-                # conversation is under way, just end it and return the last turn cleanly.
-                if completion is None:
-                    return web.json_response(
-                        dialect.error_body(f"rollout stopped: {refused}"), status=400
+        try:
+            while True:
+                try:
+                    refused = await session.refused()
+                except RolloutError as e:
+                    return self._fail(session, dialect, e)
+                except Exception as e:
+                    return self._fail(
+                        session,
+                        dialect,
+                        TaskError(f"@stop failed: {type(e).__name__}: {e}"),
+                    )
+                if refused is not None:
+                    # Refuse the first model call to halt the harness; once a simulated
+                    # conversation is under way, just end it and return the last turn cleanly.
+                    if response is None:
+                        return web.json_response(
+                            dialect.error_body(f"rollout stopped: {refused}"),
+                            status=400,
+                        )
+                    return serve(response)
+                turn = graph.prepare_turn(session.trace, prompt)
+                session.error = None
+                try:
+                    response = await session.ctx.client.get_response(
+                        dialect,
+                        body,
+                        session.ctx.model,
+                        session.ctx.sampling,
+                        headers=headers,
+                        session_id=session.trace.id,
+                        turn=turn,
+                    )
+                except OverlongPromptError:
+                    # An overlong prompt is a budget limit, not a crash: end the rollout cleanly
+                    # as a truncation — return the last turn if there is one, else refuse to halt
+                    # the harness (same shape as `refused` above).
+                    session.trace.stop("context_length")
+                    logger.debug("prompt too long: id=%s", session.trace.id)
+                    if response is None:
+                        return web.json_response(
+                            dialect.error_body("rollout stopped: context_length"),
+                            status=400,
+                        )
+                    return serve(response)
+                except RolloutError as e:
+                    # Stash the real cause; the rollout re-raises it after the harness returns.
+                    # Relay the provider's status so the harness SDK retries 5xx/429 and not 4xx.
+                    session.error = e
+                    logger.warning(
+                        "model call failed: id=%s %s: %s",
+                        session.trace.id,
+                        type(e).__name__,
+                        e,
                     )
-                return _completion_response(completion)
-            turn = graph.prepare_turn(session.trace, prompt)
-            session.error = None
-            try:
-                response = await session.ctx.client.get_response(
-                    dialect,
-                    body,
-                    session.ctx.model,
-                    session.ctx.sampling,
-                    headers=headers,
-                    session_id=session.trace.id,
-                    turn=turn,
-                )
-            except OverlongPromptError:
-                # An overlong prompt is a budget limit, not a crash: end the rollout cleanly
-                # as a truncation — return the last turn if there is one, else refuse to halt
-                # the harness (same shape as `refused` above).
-                session.trace.stop("context_length")
-                logger.debug("prompt too long: id=%s", session.trace.id)
-                if completion is None:
                     return web.json_response(
-                        dialect.error_body("rollout stopped: context_length"),
-                        status=400,
+                        dialect.error_body(str(e)),
+                        status=getattr(e, "status_code", 502),
                     )
-                return _completion_response(completion)
-            except RolloutError as e:
-                # Stash the real cause; the rollout re-raises it after the harness returns. Relay
-                # the provider's status so the harness SDK retries 5xx/429 and not 4xx.
-                session.error = e
-                logger.warning(
-                    "model call failed: id=%s %s: %s",
-                    session.trace.id,
-                    type(e).__name__,
-                    e,
-                )
-                return web.json_response(
-                    dialect.error_body(str(e)), status=getattr(e, "status_code", 502)
-                )
-            except Exception as e:  # surface to the program as an API error
-                logger.warning(
-                    "model call failed: id=%s %s: %s",
+                except Exception as e:  # surface to the program as an API error
+                    logger.warning(
+                        "model call failed: id=%s %s: %s",
+                        session.trace.id,
+                        type(e).__name__,
+                        e,
+                    )
+                    return web.json_response(dialect.error_body(str(e)), status=502)
+                logger.debug(
+                    "intercept turn: id=%s tools=%d",
                     session.trace.id,
-                    type(e).__name__,
-                    e,
-                )
-                return web.json_response(dialect.error_body(str(e)), status=502)
-            # `Response.raw` is the full native provider object, or the renderer's synthesized
-            # completion object, that the server serializes for the program.
-            completion = response.raw
-            logger.debug(
-                "intercept turn: id=%s tools=%d",
-                session.trace.id,
-                len(response.message.tool_calls or []),
-            )
-            turn.commit(response, tools)  # one node per new message;
-            # branches fall out of walking the graph (see Trace.branches / verifiers.v1.graph)
-            # Hand back to the program when the model wants a tool (the program runs it) or
-            # when there's no user simulator to keep the conversation going.
-            if response.message.tool_calls or session.user is None:
-                return _completion_response(completion)
-            try:
-                user_messages = await session.user(response.message.content or "")
-            except RolloutError as e:
-                return self._fail(session, dialect, e)
-            except Exception as e:
-                return self._fail(
-                    session,
-                    dialect,
-                    UserError(f"user simulator failed: {type(e).__name__}: {e}"),
+                    len(response.message.tool_calls or []),
                 )
-            # Inject the model turn + the simulator's user turn(s): into the wire request for the
-            # next model call (`dialect.extend`, which keeps the model turn verbatim so reasoning
-            # survives) and into the typed prompt for the trace. The simulator ends the trajectory
-            # through its task's `@stop` (e.g. a `user_finished` flag it set on `self.state`),
-            # caught by `refused()` at the top of the next iteration — the interception server holds
-            # no opinion about the state's contents.
-            body = dialect.extend(body, completion, user_messages)
-            prompt = [*prompt, response.message, *user_messages]
-            # The simulator changed the payload, so this is a new operation rather than a retry.
-            headers.popall("idempotency-key", None)
-            headers.popall("x-idempotency-key", None)
+                turn.commit(response, tools)  # one node per new message;
+                # branches fall out of walking the graph (see Trace.branches / verifiers.v1.graph)
+                # Hand back to the program when the model wants a tool (the program runs it) or
+                # when there's no user simulator to keep the conversation going.
+                if response.message.tool_calls or session.user is None:
+                    return serve(response)
+                try:
+                    user_messages = await session.user(response.message.content or "")
+                except RolloutError as e:
+                    return self._fail(session, dialect, e)
+                except Exception as e:
+                    return self._fail(
+                        session,
+                        dialect,
+                        UserError(f"user simulator failed: {type(e).__name__}: {e}"),
+                    )
+                # Inject the model turn + the simulator's user turn(s): into the wire request for
+                # the next model call (`dialect.extend`, which keeps the model turn verbatim so
+                # reasoning survives) and into the typed prompt for the trace. The simulator ends
+                # the trajectory through its task's `@stop` (e.g. a `user_finished` flag it set on
+                # `self.state`), caught by `refused()` at the top of the next iteration — the
+                # interception server holds no opinion about the state's contents.
+                body = dialect.extend(body, response.raw, user_messages)
+                prompt = [*prompt, response.message, *user_messages]
+                # The simulator changed the payload, so this is a new operation not a retry.
+                headers.popall("idempotency-key", None)
+                headers.popall("x-idempotency-key", None)
+        finally:
+            # Free the in-flight slot and unblock any coalesced retry; None signals "no servable
+            # response" (an error/refuse return above), so the waiter surfaces a retryable error.
+            # Only clear our own entry — never one a later owner may have installed.
+            if session.inflight.get(req_hash) is fut:
+                session.inflight.pop(req_hash, None)
+            if not fut.done():
+                fut.set_result(None)
 
     async def _stream(
         self,

{verifiers-0.2.1.dev42 → verifiers-0.2.1.dev44}/verifiers/v1/session.py

@@ -8,6 +8,7 @@ injects the user simulator's replies, and stashes the real failure on `error`.
 turns.
 """
 
+import asyncio
 import logging
 from collections.abc import Awaitable, Callable
 from dataclasses import dataclass, field
@@ -81,6 +82,23 @@ class RolloutSession:
     (and may swallow it, or exit non-zero), so the rollout re-raises this original error once the
     harness returns — recording the real `ProviderError` instead of a secondary `HarnessError`.
     Reset before each model turn, so a successful retry clears it."""
+    last_request: bytes | None = None
+    """Digest of the most recently served request body; with `last_response`, the replay cache
+    that keeps the message graph atomic under harness-SDK retries. A retry re-sends the
+    byte-identical request; when it matches, the interception server replays the recorded
+    response instead of re-sampling and committing a second turn — which would fork the graph
+    into a dead-end branch. Only a fully served request is cached, so a genuinely failed attempt
+    still re-runs. Turns are issued sequentially (one outstanding request at a time), so a retry
+    is always of the most recent request — keeping only the last one is sufficient and bounded."""
+    last_response: dict | None = None
+    """The response returned for `last_request`, replayed verbatim on a retry."""
+    inflight: dict[bytes, "asyncio.Future[dict | None]"] = field(default_factory=dict)
+    """Body digest -> the future of the attempt currently computing it. A retry that arrives
+    while the first attempt is still in flight (a slow turn) awaits this future instead of
+    starting a second inference — the other half of retry atomicity (with `last_response`, which
+    covers a retry after the attempt finished). Because a slow turn is coalesced rather than
+    re-sampled, retries stay safe without an inflated client timeout. The future resolves to the
+    served response, or to None if the attempt produced no servable response (error/refuse)."""
 
     async def refused(self) -> str | None:
         """The framework's limits (turns / token budget) and `@stop` checks, run before each

{verifiers-0.2.1.dev42 → verifiers-0.2.1.dev44}/verifiers/v1/trace.py

@@ -167,36 +167,37 @@ class Branch(StrictBaseModel):
         )
         return KeptTokens(ids=ids, counts=np.concatenate(counts_parts))
 
-    @property
-    def num_total_tokens(self) -> int:
-        return sum(len(n.token_ids) for n in self.nodes)
-
     @property
     def usage(self) -> Usage | None:
         return Usage.aggregate(n.usage for n in self.nodes if n.usage is not None)
 
     @property
-    def num_input_tokens(self) -> int:
-        """Final-turn prompt size, falling back to provider usage without token IDs."""
-        last_completion = next(
-            (sum(n.mask) for n in reversed(self.nodes) if any(n.mask)), 0
-        )
-        token_len = self.num_total_tokens - last_completion
-        if token_len:
-            return token_len
-        last = next(
+    def last_usage(self) -> Usage | None:
+        """Provider usage from the final model call on this branch — the full context it saw."""
+        return next(
             (n.usage for n in reversed(self.nodes) if n.usage is not None), None
         )
-        return last.input_tokens if last else 0
+
+    @property
+    def num_total_tokens(self) -> int:
+        """Final sequence length: the last call's prompt + completion. Earlier turns' context
+        is already contained in that prompt, so re-sent tokens are counted once rather than
+        summed per turn."""
+        last = self.last_usage
+        return last.total_tokens if last is not None else 0
 
     @property
     def num_output_tokens(self) -> int:
-        """Sampled tokens, falling back to provider usage without token IDs."""
-        token_len = sum(sum(n.mask) for n in self.nodes)
-        if token_len:
-            return token_len
+        """Every model-generated token across all turns (completions, reasoning included)."""
         usage = self.usage
-        return usage.completion_tokens if usage else 0
+        return usage.completion_tokens if usage is not None else 0
+
+    @property
+    def num_input_tokens(self) -> int:
+        """Tokens fed to the model, counted once: the final sequence minus everything the model
+        generated (i.e. system + user + tool inputs). Not the last prompt — re-sent context is
+        not double-counted."""
+        return self.num_total_tokens - self.num_output_tokens
 
 
 _NODE_DUMP_EXCLUDE: dict = {
@@ -282,17 +283,17 @@ class Trace(StrictBaseModel, Generic[DataT, StateT]):
 
     @property
     def num_input_tokens(self) -> int:
-        """Final-turn prompt sizes summed across training branches."""
+        """Fed-in tokens (system + user + tool), counted once, summed across branches."""
         return sum(branch.num_input_tokens for branch in self.branches)
 
     @property
     def num_output_tokens(self) -> int:
-        """Model-sampled tokens summed across training branches."""
+        """Model-generated tokens across all turns, summed across branches."""
         return sum(branch.num_output_tokens for branch in self.branches)
 
     @property
     def num_total_tokens(self) -> int:
-        """Sequence lengths summed across training branches for token batching."""
+        """Final sequence lengths (last prompt + completion) summed across branches."""
         return sum(branch.num_total_tokens for branch in self.branches)
 
     @property