verifiers 0.1.15.dev425__tar.gz → 0.1.15.dev427__tar.gz

{verifiers-0.1.15.dev425 → verifiers-0.1.15.dev427}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.1.15.dev425
+Version: 0.1.15.dev427
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.1.15.dev425 → verifiers-0.1.15.dev427}/verifiers/v1/cli/dashboard/eval.py

@@ -209,15 +209,24 @@ def Overview(config: EvalConfig) -> Table:
 
 
 def Progress(
-    rollouts: list[Rollout], start: float, page: tuple[int, int] | None = None
+    rollouts: list[Rollout],
+    start: float,
+    page: tuple[int, int] | None = None,
+    finished: list[Trace] | None = None,
 ) -> Group:
-    done = [r.trace for r in rollouts if r.phase == Phase.DONE]  # fully scored
+    # On resume, `finished` holds the kept on-disk rollouts (reloaded as finished traces); count
+    # them alongside this session's so progress, reward, err, and the breakdown cover the whole
+    # run. `rollouts` is only this session's (owed) work, so the total adds the kept ones back.
+    done = (finished or []) + [
+        r.trace for r in rollouts if r.phase == Phase.DONE
+    ]  # fully scored
+    total = len(finished or []) + len(rollouts)
     # Headline reward = mean over non-errored; when any errored, `format_mean` appends the
     # global avg (errored count as 0) in parens. `err` is the share that errored.
     reward = format_mean(done, lambda t: t.reward)
     err = f"{sum(t.has_error for t in done) / len(done):.2f}" if done else "—"
     stats = (
-        f"{len(done)}/{len(rollouts)} · {format_time(time.time() - start)} · "
+        f"{len(done)}/{total} · {format_time(time.time() - start)} · "
         f"reward {reward} · err {err}"
     )
     if page is not None:  # overflowing — show which page, and that the arrows page
@@ -226,7 +235,7 @@ def Progress(
     row.add_column(ratio=1)  # bar stretches to fill the width left of the stats
     row.add_column(justify="right", no_wrap=True)
     row.add_row(
-        ProgressBar(total=len(rollouts) or 1, completed=len(done)),
+        ProgressBar(total=total or 1, completed=len(done)),
         Text(stats),
     )
     breakdown = _breakdown(done)
@@ -561,7 +570,11 @@ def _paginate(
 
 
 def _render(
-    rollouts: list[Rollout], config: EvalConfig, start: float, pager: Pager
+    rollouts: list[Rollout],
+    config: EvalConfig,
+    start: float,
+    pager: Pager,
+    finished: list[Trace] | None = None,
 ) -> Group:
     now = time.time()
     warning = _warning(config)
@@ -570,10 +583,15 @@ def _render(
     # Measure the fixed top (header + progress + rule) so the rollout rows fill the rest of the
     # screen; page through them (timer, or the left/right arrows) when they'd overflow (rich would
     # otherwise truncate).
-    top = Group(header, Progress(rollouts, start), Rule(style="dim"))
+    top = Group(header, Progress(rollouts, start, finished=finished), Rule(style="dim"))
     rows_per_page = max(1, _CONSOLE.size.height - len(_CONSOLE.render_lines(top)) - 1)
     page_groups, index, count = _paginate(_groups(rollouts), rows_per_page, pager, now)
-    progress = Progress(rollouts, start, page=(index + 1, count) if count > 1 else None)
+    progress = Progress(
+        rollouts,
+        start,
+        page=(index + 1, count) if count > 1 else None,
+        finished=finished,
+    )
     return Group(
         header,
         progress,
@@ -583,11 +601,18 @@ def _render(
 
 
 @contextlib.asynccontextmanager
-async def dashboard(rollouts: list[Rollout], config: EvalConfig, start: float):
+async def dashboard(
+    rollouts: list[Rollout],
+    config: EvalConfig,
+    start: float,
+    finished: list[Trace] | None = None,
+):
     """Refresh the live eval view until the `with` block exits, then a final frame. Left/right
-    arrows page through rollout rows when they overflow the screen."""
+    arrows page through rollout rows when they overflow the screen. On resume, `finished` carries
+    the kept on-disk rollouts (reloaded as finished traces) so the counts and scores cover the
+    whole run, not just this session's re-run rollouts."""
     pager = Pager()
     async with live_view(
-        lambda: _render(rollouts, config, start, pager), on_key=pager.on_key
+        lambda: _render(rollouts, config, start, pager, finished), on_key=pager.on_key
     ):
         yield

{verifiers-0.1.15.dev425 → verifiers-0.1.15.dev427}/verifiers/v1/cli/eval/resume.py

@@ -16,7 +16,12 @@ from pathlib import Path
 
 from pydantic_core import from_json
 
+from verifiers.v1.cli.output import read_traces
 from verifiers.v1.configs.eval import EvalConfig
+from verifiers.v1.state import state_cls
+from verifiers.v1.task import WireTask
+from verifiers.v1.taskset import Taskset
+from verifiers.v1.trace import Trace
 
 
 def split_resume(argv: list[str]) -> tuple[Path | None, list[str]]:
@@ -116,6 +121,14 @@ def rewrite_results(resume_dir: Path, keep: list[int]) -> None:
     tmp.replace(path)
 
 
+def load_kept(resume_dir: Path, taskset: Taskset) -> list[Trace]:
+    """Reload the kept (good) traces as finished `Trace`s, so a resumed run's live dashboard counts
+    them toward the whole run (progress, reward, err, and the usage/time breakdown). Call *after*
+    `rewrite_results`, which leaves only the kept rows on disk. `WireTask` reads any taskset's saved
+    task without a runtime or its `Task` type (mirrors `replay`)."""
+    return read_traces(resume_dir, Trace[WireTask, state_cls(type(taskset))])
+
+
 def nothing_to_resume_msg(resume_dir: Path, num_tasks: int, num_rollouts: int) -> str:
     """The message shown (and then exit 0 - the run is already complete) when every selected
     rollout already completed without error."""

{verifiers-0.1.15.dev425 → verifiers-0.1.15.dev427}/verifiers/v1/cli/eval/runner.py

@@ -41,6 +41,10 @@ async def run_eval(env: Environment, config: EvalConfig) -> list[Trace]:
     # run only the owed rollouts. One lock serializes worker-thread appends from concurrent
     # rollouts while keeping large trace serialization off the event loop.
     owed: dict[str, int] | None = None
+    # On resume, the kept (good) on-disk rollouts, reloaded as finished traces so the live
+    # dashboard counts the whole run (progress, reward, err, usage/time) rather than only this
+    # session's re-run rollouts. Only the --rich dashboard reads them, so skip the load otherwise.
+    finished: list[Trace] = []
     if config.resume is not None:
         group = bool(discover_decorated(env.taskset, "group_reward"))
         keep, owed = resume.plan(
@@ -51,6 +55,8 @@ async def run_eval(env: Environment, config: EvalConfig) -> list[Trace]:
             raise SystemExit(0)
         tasks = [task for task in tasks if owed.get(task.idx)]
         resume.rewrite_results(out, keep)
+        if config.rich:
+            finished = resume.load_kept(out, env.taskset)
         logger.info(
             "resuming %s: %d task(s), %d rollout(s) owed",
             out,
@@ -84,7 +90,7 @@ async def run_eval(env: Environment, config: EvalConfig) -> list[Trace]:
         ]
         rollouts = [rollout for episode in episodes for rollout in episode.rollouts]
         display = (
-            dashboard(rollouts, config, start)
+            dashboard(rollouts, config, start, finished=finished)
             if config.rich
             else contextlib.nullcontext()
         )

{verifiers-0.1.15.dev425 → verifiers-0.1.15.dev427}/verifiers/v1/clients/client.py

@@ -67,7 +67,13 @@ class Client(ABC):
         generates and cannot stream."""
         raise NotImplementedError(f"{type(self).__name__} does not support streaming")
 
-    async def relay_aux(self, dialect: Dialect, route: str, body: dict) -> dict:
+    async def relay_aux(
+        self,
+        dialect: Dialect,
+        route: str,
+        body: dict,
+        headers: Mapping[str, str] | None = None,
+    ) -> dict:
         """Relay a non-model-turn side request (an `aux_route`, e.g. Anthropic's `count_tokens`)
         verbatim to the provider and return its JSON. Only the relay (eval) client supports it."""
         raise NotImplementedError(f"{type(self).__name__} does not relay aux routes")

{verifiers-0.1.15.dev425 → verifiers-0.1.15.dev427}/verifiers/v1/clients/eval.py

@@ -205,12 +205,18 @@ class EvalClient(Client):
             close=resp.aclose,
         )
 
-    async def relay_aux(self, dialect: Dialect, route: str, body: dict) -> dict:
+    async def relay_aux(
+        self,
+        dialect: Dialect,
+        route: str,
+        body: dict,
+        headers: Mapping[str, str] | None = None,
+    ) -> dict:
         # A side request (e.g. count_tokens): relay its native JSON and return the provider JSON.
         resp = await self._request(
             self.base_url + route,
             body,
-            self._headers(dialect, None, None),
+            self._headers(dialect, headers, None),
         )
         return from_json(resp.content)
 

{verifiers-0.1.15.dev425 → verifiers-0.1.15.dev427}/verifiers/v1/interception/server.py

@@ -543,7 +543,7 @@ class InterceptionServer:
         logger.debug("intercept aux %s: id=%s", route, session.trace.id)
         try:
             result = await session.ctx.client.relay_aux(
-                dialect, route, await request.json()
+                dialect, route, await request.json(), headers=request.headers
             )
         except RolloutError as e:
             # An aux call isn't a model turn, so don't clobber a pending turn error.