verifiers 0.2.1.dev33__tar.gz → 0.2.1.dev35__tar.gz

{verifiers-0.2.1.dev33 → verifiers-0.2.1.dev35}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.1.dev33
+Version: 0.2.1.dev35
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.2.1.dev33 → verifiers-0.2.1.dev35}/verifiers/v1/cli/dashboard/replay.py

@@ -6,13 +6,12 @@ from dataclasses import dataclass
 
 from rich.console import Group
 from rich.markup import escape
-from rich.progress_bar import ProgressBar
 from rich.rule import Rule
 from rich.table import Table
 from rich.text import Text
 
 from verifiers.v1.cli.dashboard.base import live_view
-from verifiers.v1.cli.dashboard.validate import TaskProgress
+from verifiers.v1.cli.dashboard.validate import Progress, TaskProgress
 from verifiers.v1.utils.format import format_time
 
 
@@ -33,30 +32,21 @@ _STYLE = {
 _MARK_WIDTH = max(len(state) for state in _STYLE)
 _MARK = {state: escape(f"[{state:<{_MARK_WIDTH}}]") for state in _STYLE}
 _DONE = ("scored", "error", "skipped")
+_OUTCOMES = {
+    "scored": ("scored", _STYLE["scored"]),
+    "error": ("failed", _STYLE["error"]),
+    "skipped": ("skipped", _STYLE["skipped"]),
+}
 
 
 def _render(
     states: list[TaskProgress], taskset_name: str, source: str, out: str, start: float
 ) -> Group:
-    done = [s for s in states if s.state in _DONE]
-    scored = sum(1 for s in done if s.state == "scored")
-    skipped = sum(1 for s in done if s.state == "skipped")
     overview = Table.grid(padding=(0, 2))
     overview.add_column(style="dim")
     overview.add_column()
-    overview.add_row("replay", f"{taskset_name}  ·  {source}")
-    overview.add_row("output", out)
-
-    stats = (
-        f" {len(done)}/{len(states)} · {format_time(time.time() - start)} · "
-        f"scored {scored} · failed {len(done) - scored - skipped} · skipped {skipped}"
-    )
-    progress = Table.grid()
-    progress.add_column()
-    progress.add_column()
-    progress.add_row(
-        ProgressBar(total=len(states) or 1, completed=len(done), width=32), Text(stats)
-    )
+    overview.add_row("replay", Text(f"{taskset_name}  ·  {source}", overflow="fold"))
+    overview.add_row("output", Text(out, overflow="fold"))
 
     now = time.time()
     rows = Table.grid(expand=True, padding=(0, 1))
@@ -80,7 +70,7 @@ def _render(
             elapsed,
             style=_STYLE[s.state],
         )
-    return Group(overview, progress, Rule(style="dim"), rows)
+    return Group(overview, Progress(states, start, _OUTCOMES), Rule(style="dim"), rows)
 
 
 @contextlib.asynccontextmanager

{verifiers-0.2.1.dev33 → verifiers-0.2.1.dev35}/verifiers/v1/cli/dashboard/validate.py

@@ -30,6 +30,8 @@ _MARK_WIDTH = max(len(state) for state in _STYLE)
 # literal: Rich parses `[name]` in a cell as markup and would otherwise drop it.
 _MARK = {state: escape(f"[{state:<{_MARK_WIDTH}}]") for state in _STYLE}
 _DONE = ("valid", "invalid", "error", "timeout")
+# State -> (visible label, color); insertion order is the summary order.
+_OUTCOMES = {state: (state, _STYLE[state]) for state in _DONE}
 
 
 @dataclass
@@ -49,13 +51,17 @@ def Overview(config: ValidateConfig) -> Table:
     return grid
 
 
-def Progress(states: list[TaskProgress], start: float) -> Table:
-    done = [s for s in states if s.state in _DONE]
+def Progress(
+    states: list[TaskProgress],
+    start: float,
+    outcomes: dict[str, tuple[str, str]] = _OUTCOMES,
+) -> Table:
+    done = [s for s in states if s.state in outcomes]
     counts = Counter(s.state for s in done)
     stats = Text(f" {len(done)}/{len(states)} · {format_time(time.time() - start)}")
-    for state in _DONE:
+    for state, (label, style) in outcomes.items():
         stats.append(" · ")
-        stats.append(f"{state} {counts[state]}", style=_STYLE[state])
+        stats.append(f"{label} {counts[state]}", style=style)
     row = Table.grid()
     row.add_column()
     row.add_column()

{verifiers-0.2.1.dev33 → verifiers-0.2.1.dev35}/verifiers/v1/graph.py

@@ -36,6 +36,7 @@ from verifiers.v1.types import (
     Response,
     StrictBaseModel,
     TextContentPart,
+    Tool,
     ToolMessage,
     Usage,
 )
@@ -341,8 +342,10 @@ class PendingTurn:
             for span in tail_spans
         ]
 
-    def commit(self, response: Response) -> None:
+    def commit(self, response: Response, tools: list[Tool] | None = None) -> None:
         _commit_turn(self, response)
+        if tools:
+            self.trace.tools = tools
 
 
 def prepare_turn(trace: Trace, prompt: list[Message]) -> PendingTurn:

{verifiers-0.2.1.dev33 → verifiers-0.2.1.dev35}/verifiers/v1/interception/pool.py

@@ -62,14 +62,11 @@ class StaticInterceptionPool(Interception):
 
     @asynccontextmanager
     async def acquire(self, session: RolloutSession) -> AsyncIterator[Slot]:
-        # min + register have no await between them, so concurrent acquires can't all
-        # land on the same "least-loaded" server.
+        # server.acquire registers before its first yield, so concurrent acquires see the
+        # updated load before choosing their own least-loaded server.
         server = min(self.servers, key=lambda s: s.load)
-        secret = server.register(session)
-        try:
-            yield server.base_url, secret
-        finally:
-            server.unregister(secret)
+        async with server.acquire(session) as slot:
+            yield slot
 
 
 class ElasticInterceptionPoolConfig(BaseInterceptionConfig):

{verifiers-0.2.1.dev33 → verifiers-0.2.1.dev35}/verifiers/v1/interception/server.py

@@ -335,14 +335,8 @@ class InterceptionServer(Interception):
                 session.trace.id,
                 len(response.message.tool_calls or []),
             )
-            turn.commit(response)  # one node per new message;
+            turn.commit(response, tools)  # one node per new message;
             # branches fall out of walking the graph (see Trace.branches / verifiers.v1.graph)
-            if tools:
-                # Persist the advertised tools now that the model actually saw them (see
-                # `Trace.tools`). Only a tools-bearing turn updates it (truthy, so an empty
-                # parse or a trailing tool-less call never erases the schemas the recorded
-                # tool calls need).
-                session.trace.tools = tools
             # Hand back to the program when the model wants a tool (the program runs it) or
             # when there's no user simulator to keep the conversation going.
             if response.message.tool_calls or session.user is None:
@@ -489,9 +483,7 @@ class InterceptionServer(Interception):
         try:
             if parser_error is not None:
                 raise parser_error
-            turn.commit(parser.finish())
-            if tools:  # record only on commit, like the non-streaming path
-                session.trace.tools = tools
+            turn.commit(parser.finish(), tools)
             logger.debug("intercept stream turn: id=%s", session.trace.id)
         finally:
             # Release the withheld events only now — after the commit — then close.