verifiers 0.1.15.dev393__tar.gz → 0.1.15.dev395__tar.gz

{verifiers-0.1.15.dev393 → verifiers-0.1.15.dev395}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.1.15.dev393
+Version: 0.1.15.dev395
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.1.15.dev393 → verifiers-0.1.15.dev395}/verifiers/v1/cli/dashboard/eval.py

@@ -167,17 +167,21 @@ def Progress(
 
 def _breakdown(done: list[Trace]) -> Table | None:
     """The per-component view under the headline (summed) reward: a `rewards` row of each named
-    `@reward` contribution and a `metrics` row of each `@metric`, laid out like the Overview (dim
-    label column). Each component is formatted exactly like the headline reward — the
-    error-corrected mean, with the global mean (an errored trace's value counting as 0) in parens
-    when some errored (see `format_mean`). `None` when nothing has scored yet, so the bar shows
-    alone."""
-    if not any(not t.has_error for t in done):
-        return None
+    `@reward` contribution and a `metrics` row of each `@metric`, then a `usage` row (tokens and
+    cost summed over completed rollouts) and a `time` row (each phase averaged over the rollouts
+    that have it timed), laid out like the Overview (dim label column). Reward/metric components
+    are error-corrected means, with the global mean (an errored trace's value counting as 0) in
+    parens when some errored (see `format_mean`); they're skipped when every rollout errored (no
+    clean mean to show), while usage/time still appear — those resources were spent regardless.
+    `None` when no rollout has completed."""
     grid = Table.grid(padding=(0, 2))
     grid.add_column(style="dim", min_width=_LABEL_WIDTH)
     grid.add_column()
-    for label, source in (("rewards", "rewards"), ("metrics", "metrics")):
+    # rewards/metrics are error-corrected means — skip when every rollout errored (no clean mean to
+    # show); usage/time below still cover errored rollouts (their resources were spent regardless).
+    has_clean = any(not t.has_error for t in done)
+    score_rows = (("rewards", "rewards"), ("metrics", "metrics")) if has_clean else ()
+    for label, source in score_rows:
         # every key seen across traces, first-seen order (a trace records only the functions
         # that ran for it, so keys can vary)
         names: list[str] = []
@@ -190,6 +194,53 @@ def _breakdown(done: list[Trace]) -> Table | None:
             for name in names
         ]
         grid.add_row(label, "  ·  ".join(segments))
+
+    # Resource use over every completed rollout (errored ones still spent tokens/time): tokens and
+    # cost are summed; each timing phase is averaged over the rollouts that have it timed (averaged
+    # per phase rather than summed, since phases run concurrently across rollouts).
+    total_in = total_out = total_cached = total_reasoning = 0
+    total_cost = 0.0
+    have_cost = have_cached = have_reasoning = False
+    phase_secs: dict[str, float] = {}
+    phase_count: dict[str, int] = {}
+    for trace in done:
+        prompt, completion, cached, reasoning, _ = _tokens(trace)
+        total_in += prompt
+        total_out += completion
+        if cached is not None:
+            total_cached += cached
+            have_cached = True
+        if reasoning is not None:
+            total_reasoning += reasoning
+            have_reasoning = True
+        if trace.usage is not None and trace.usage.cost is not None:
+            total_cost += trace.usage.cost
+            have_cost = True
+        for phase in ("setup", "generation", "finalize", "scoring"):
+            span = getattr(trace.timing, phase)
+            if span.end:  # phase was timed for this rollout
+                phase_secs[phase] = phase_secs.get(phase, 0.0) + span.duration
+                phase_count[phase] = phase_count.get(phase, 0) + 1
+    if total_in or total_out or have_cost or have_cached or have_reasoning:
+        tokens = f"{format_count(total_in)}/{format_count(total_out)} tokens"
+        details = []
+        if have_cached:
+            details.append(f"{format_count(total_cached)} cached")
+        if have_reasoning:
+            details.append(f"{format_count(total_reasoning)} reasoning")
+        if details:
+            tokens += f" ({', '.join(details)})"
+        usage = [tokens]
+        if have_cost:
+            usage.append(f"total {format_cost_usd(total_cost)}")
+        grid.add_row("usage", "  ·  ".join(usage))
+    time_segments = [
+        f"{phase} {format_time(phase_secs[phase] / phase_count[phase])}"
+        for phase in ("setup", "generation", "finalize", "scoring")
+        if phase_count.get(phase)
+    ]
+    if time_segments:
+        grid.add_row("time", "  ·  ".join(time_segments))
     return grid if grid.row_count else None
 
 
@@ -277,10 +328,10 @@ def Rows(groups: list[list[Rollout]], now: float, runtime_type: str) -> Table:
             if prompt or completion:
                 tokens = f"{format_count(prompt)}/{format_count(completion)} tokens"
                 details = []
-                if reasoning is not None:
-                    details.append(f"{format_count(reasoning)} reasoning")
                 if cached is not None:
                     details.append(f"{format_count(cached)} cached")
+                if reasoning is not None:
+                    details.append(f"{format_count(reasoning)} reasoning")
                 if details:
                     tokens += f" ({', '.join(details)})"
             left = [

{verifiers-0.1.15.dev393 → verifiers-0.1.15.dev395}/verifiers/v1/dialects/base.py

@@ -151,6 +151,14 @@ class Dialect(ABC, Generic[ReqT, RespT]):
         """Whether the request asks for a streamed (SSE) response."""
         return bool(body.get("stream"))
 
+    def is_terminal_event(self, chunk: bytes) -> bool:
+        """Whether this complete SSE event ends the model's turn for the client. The
+        interception server withholds the terminal event (and anything after it) until the
+        turn is recorded, so a client that ends its turn on it can't race ahead to scoring
+        with the turn still uncommitted. Defaults to the `[DONE]` sentinel; a dialect whose
+        client ends on an earlier event (e.g. Responses' `response.completed`) overrides this."""
+        return is_sse_done_event(chunk)
+
     def error_body(self, message: str) -> dict:
         """An error payload in this format's error shape (OpenAI by default)."""
         return {"error": {"message": message, "type": "invalid_request_error"}}

{verifiers-0.1.15.dev393 → verifiers-0.1.15.dev395}/verifiers/v1/dialects/responses.py

@@ -47,6 +47,13 @@ from verifiers.v1.types import (
 )
 
 FINAL_EVENTS = ("response.completed", "response.incomplete", "response.failed")
+# Byte markers for the terminal event types above, in both compact and spaced JSON, so the
+# interception server can cheaply spot the turn-ending event without parsing each delta.
+_TERMINAL_MARKERS = tuple(
+    marker.encode()
+    for event in FINAL_EVENTS
+    for marker in (f'"type":"{event}"', f'"type": "{event}"')
+)
 # Sampling knobs the eval owns, in this format's shape (Responses uses `max_output_tokens`).
 _SAMPLING_KEYS = frozenset({"temperature", "top_p", "max_output_tokens", "max_tokens"})
 
@@ -263,6 +270,11 @@ class ResponsesDialect(Dialect[dict, OpenAIResponse]):
     upstream_path = "/responses"
     response_type = OpenAIResponse
 
+    def is_terminal_event(self, chunk: bytes) -> bool:
+        # A Responses client (e.g. codex) ends its turn on `response.completed`, before the
+        # trailing `[DONE]`, so the turn-ending event is the final event, not the sentinel.
+        return any(marker in chunk for marker in _TERMINAL_MARKERS)
+
     def parse_request(self, body: dict) -> tuple[Messages, list[Tool] | None]:
         prompt: Messages = []
         if instructions := body.get("instructions"):

{verifiers-0.1.15.dev393 → verifiers-0.1.15.dev395}/verifiers/v1/interception/server.py

@@ -472,6 +472,11 @@ class InterceptionServer:
         ready = asyncio.Event()
         producer = asyncio.create_task(_queue_chunks(reply.chunks, queue, ready))
         parser_error: Exception | None = None
+        # SSE events from the turn-ending one onward (the terminal event and any trailing
+        # `[DONE]`), withheld until the turn is committed: a client that ends its turn on the
+        # terminal event (e.g. codex on `response.completed`) would otherwise reach scoring
+        # with the turn still unrecorded.
+        deferred: list[bytes] = []
         try:
             await resp.prepare(request)
             while True:
@@ -487,11 +492,20 @@ class InterceptionServer:
                 if chunk is None:
                     await producer
                     break
+                if deferred or dialect.is_terminal_event(chunk):
+                    if parser_error is None:
+                        try:
+                            if on_done is not None and is_sse_done_event(chunk):
+                                on_done()
+                            feed_event(chunk)
+                        except Exception as e:
+                            parser_error = e
+                    # forwarded after the turn is committed, below
+                    deferred.append(chunk)
+                    continue
                 await resp.write(chunk)
                 if parser_error is None:
                     try:
-                        if on_done is not None and is_sse_done_event(chunk):
-                            on_done()
                         feed_event(chunk)
                     except Exception as e:
                         parser_error = e
@@ -511,7 +525,10 @@ class InterceptionServer:
             turn.commit(parser.finish())
             logger.debug("intercept stream turn: id=%s", session.trace.id)
         finally:
+            # Release the withheld events only now — after the commit — then close.
             with contextlib.suppress(ConnectionResetError):
+                for event in deferred:
+                    await resp.write(event)
                 await resp.write_eof()
         return resp