verifiers 0.1.15.dev441__tar.gz → 0.1.15.dev442__tar.gz

{verifiers-0.1.15.dev441 → verifiers-0.1.15.dev442}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.1.15.dev441
+Version: 0.1.15.dev442
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.1.15.dev441 → verifiers-0.1.15.dev442}/verifiers/v1/cli/dashboard/eval.py

@@ -231,15 +231,11 @@ def Progress(
     rollouts: list[Rollout],
     start: float,
     page: tuple[int, int] | None = None,
-    finished: list[Trace] | None = None,
 ) -> Group:
-    # On resume, `finished` holds the kept on-disk rollouts (reloaded as finished traces); count
-    # them alongside this session's so progress, reward, err, and the breakdown cover the whole
-    # run. `rollouts` is only this session's (owed) work, so the total adds the kept ones back.
-    done = (finished or []) + [
-        r.trace for r in rollouts if r.phase == Phase.DONE
-    ]  # fully scored
-    total = len(finished or []) + len(rollouts)
+    # On resume, `rollouts` includes the previous session's kept rollouts (as `Finished`), so
+    # progress, reward, err, and the breakdown cover the whole run, not just this session's.
+    done = [r.trace for r in rollouts if r.phase == Phase.DONE]  # fully scored
+    total = len(rollouts)
     # Headline reward = mean over non-errored; when any errored, `format_mean` appends the
     # global avg (errored count as 0) in parens. `err` is the share that errored.
     reward = format_mean(done, lambda t: t.reward)
@@ -594,7 +590,6 @@ def _render(
     config: EvalConfig,
     start: float,
     pager: Pager,
-    finished: list[Trace] | None = None,
     push: "PushState | None" = None,
 ) -> Group:
     now = time.time()
@@ -606,7 +601,7 @@ def _render(
     # rows fill what's left; page through them (timer / arrows) when they'd overflow (else rich
     # truncates).
     footer = _push_footer(push)
-    top = Group(header, Progress(rollouts, start, finished=finished), Rule(style="dim"))
+    top = Group(header, Progress(rollouts, start), Rule(style="dim"))
     reserved = len(_CONSOLE.render_lines(top))
     if footer is not None:
         reserved += len(_CONSOLE.render_lines(footer))
@@ -616,7 +611,6 @@ def _render(
         rollouts,
         start,
         page=(index + 1, count) if count > 1 else None,
-        finished=finished,
     )
     parts = [
         header,
@@ -634,18 +628,17 @@ async def dashboard(
     rollouts: list[Rollout],
     config: EvalConfig,
     start: float,
-    finished: list[Trace] | None = None,
     push: "PushState | None" = None,
 ):
     """Refresh the live eval view until the `with` block exits, then a final frame. Left/right
-    arrows page through rollout rows when they overflow the screen. On resume, `finished` carries
-    the kept on-disk rollouts (reloaded as finished traces) so the counts and scores cover the
-    whole run, not just this session's re-run rollouts. `push` is the shared `--push` status the
-    view renders as a line under the rollouts once the upload starts (dim -> white URL / red error),
-    updated by the caller as the inline upload runs and lands."""
+    arrows page through rollout rows when they overflow the screen. On resume, `rollouts`
+    includes the previous session's kept rollouts (as finished ones), so the rows, counts, and
+    scores cover the whole run, not just this session's re-run rollouts. `push` is the shared
+    `--push` status the view renders as a line under the rollouts once the upload starts (dim ->
+    white URL / red error), updated by the caller as the inline upload runs and lands."""
     pager = Pager()
     async with live_view(
-        lambda: _render(rollouts, config, start, pager, finished, push),
+        lambda: _render(rollouts, config, start, pager, push),
         on_key=pager.on_key,
     ):
         yield

verifiers-0.1.15.dev442/verifiers/v1/cli/eval/resume.py

@@ -0,0 +1,119 @@
+"""Resume an interrupted eval: reload its finished rollouts and run only the rest.
+
+A run writes `config.toml` + `traces.jsonl` into its output dir. `--resume <dir>` reloads
+that config verbatim (so it takes no other flags) and writes back into the same dir. `load`
+brings the good saved rollouts back into memory and re-runs only what's still owed: the
+*missing* rollouts (never written — the run was interrupted) and the *errored* ones (written
+with an error; dropped and redone). The loaded traces rejoin the run everywhere — counted,
+displayed, pushed, and printed alongside this session's — so a resumed run picks up exactly
+where the interrupted one stopped. A group-scored taskset is resumed a whole task at a time
+(its rollouts are scored together), so any task that isn't fully complete is redone from
+scratch.
+"""
+
+import json
+import tomllib
+from collections import defaultdict
+from pathlib import Path
+
+from pydantic_core import from_json
+
+from verifiers.v1.cli.output import CONFIG_FILE, TRACES_FILE
+from verifiers.v1.configs.eval import EvalConfig
+from verifiers.v1.rollout import Phase, Rollout
+from verifiers.v1.trace import Trace, WireTrace
+
+
+def split_resume(argv: list[str]) -> tuple[Path | None, list[str]]:
+    """Pull `--resume <dir>` / `--resume=<dir>` out of argv, returning (dir, the other args).
+    The caller rejects any leftover args, since resume re-runs the saved config verbatim."""
+    for i, arg in enumerate(argv):
+        if arg == "--resume":
+            if i + 1 >= len(argv):
+                raise SystemExit(
+                    "--resume needs an output dir: uv run eval --resume <dir>"
+                )
+            return Path(argv[i + 1]), argv[:i] + argv[i + 2 :]
+        if arg.startswith("--resume="):
+            return Path(arg.split("=", 1)[1]), argv[:i] + argv[i + 1 :]
+    return None, argv
+
+
+def load_resume_config(resume_dir: Path) -> EvalConfig:
+    """Rebuild the run's `EvalConfig` from its saved `config.toml`, pointed back at its own
+    output dir so the resumed rollouts append to the same `traces.jsonl`."""
+    config_path = resume_dir / CONFIG_FILE
+    if not config_path.exists():
+        raise SystemExit(
+            f"--resume: no config.toml in {resume_dir} - not an eval output dir"
+        )
+    config = EvalConfig.model_validate(tomllib.loads(config_path.read_text()))
+    config.resume = resume_dir
+    config.output_dir = resume_dir
+    return config
+
+
+class Finished(Rollout):
+    """A finished rollout reloaded from a previous session (see `load`): carries just the
+    surface the dashboard reads (`trace`/`task`/`phase`/`runtime`), so a resumed run's kept
+    rollouts render exactly like this session's."""
+
+    def __init__(self, trace: Trace) -> None:
+        self.trace = trace
+        self.task = trace.task
+        self.phase = Phase.DONE
+        self.runtime = None
+
+
+def load(
+    resume_dir: Path, selected_idxs: list[int], num_rollouts: int, group: bool
+) -> tuple[list[Trace], dict[int, int]]:
+    """Load the good saved rollouts back into memory as finished traces and diff them against
+    the run's target (`num_rollouts` per selected task): returns (the kept traces, rollouts
+    owed per task idx). An errored trace is dropped and re-run; a group-scored task is kept
+    only if fully complete, else its whole group is redone. Rewrites `traces.jsonl` to just
+    the kept rows — verbatim, via a temp file + atomic rename, so an interrupted resume can't
+    corrupt the prior good results — and the resumed rollouts then append. `WireTrace` reads
+    any taskset's saved traces without importing it."""
+    path = resume_dir / TRACES_FILE
+    selected = set(selected_idxs)
+    good: dict[int, list[bytes]] = defaultdict(list)
+    if path.exists():
+        with path.open("rb") as results:
+            for line in results:
+                if not line.strip():
+                    continue
+                try:
+                    row = from_json(line)
+                except ValueError:
+                    row = json.loads(line)
+                idx = row["task"]["idx"]
+                if (
+                    idx in selected
+                    and not row.get("errors")
+                    and len(good[idx]) < num_rollouts
+                ):
+                    good[idx].append(line if line.endswith(b"\n") else line + b"\n")
+    keep: list[bytes] = []
+    owed: dict[int, int] = {}
+    for idx in selected_idxs:
+        rows = good.get(idx, [])
+        if group and len(rows) < num_rollouts:
+            owed[idx] = num_rollouts  # re-run the whole group; keep none of it
+            continue
+        keep.extend(rows)
+        if missing := num_rollouts - len(rows):
+            owed[idx] = missing
+    tmp = path.with_suffix(".jsonl.tmp")
+    tmp.write_bytes(b"".join(keep))
+    tmp.replace(path)
+    return [WireTrace.model_validate_json(line) for line in keep], owed
+
+
+def nothing_to_resume_msg(resume_dir: Path, num_tasks: int, num_rollouts: int) -> str:
+    """The message shown (and then exit 0 - the run is already complete) when every selected
+    rollout already completed without error."""
+    return (
+        f"nothing to resume in {resume_dir}: all {num_tasks}x{num_rollouts} rollouts "
+        f"already completed without error"
+    )

{verifiers-0.1.15.dev441 → verifiers-0.1.15.dev442}/verifiers/v1/cli/eval/runner.py

@@ -37,26 +37,22 @@ async def run_eval(env: Environment, config: EvalConfig) -> list[Trace]:
     )
     out = output_path(config)
     # Write config.toml up front, then persist each trace as it completes (so the results are
-    # durable mid-run, not only at the end). On resume, keep the saved config + good traces and
-    # run only the owed rollouts. One lock serializes worker-thread appends from concurrent
-    # rollouts while keeping large trace serialization off the event loop.
+    # durable mid-run, not only at the end). One lock serializes worker-thread appends from
+    # concurrent rollouts while keeping large trace serialization off the event loop.
     owed: dict[str, int] | None = None
-    # On resume, the kept (good) on-disk rollouts, reloaded as finished traces so the live
-    # dashboard counts the whole run (progress, reward, err, usage/time) rather than only this
-    # session's re-run rollouts. Only the --rich dashboard reads them, so skip the load otherwise.
+    # On resume, the kept (good) on-disk rollouts rejoin the run as finished traces: displayed,
+    # returned, pushed, and printed alongside this session's, with only the owed rollouts re-run
+    # — so the resumed run is indistinguishable from one that was never interrupted.
     finished: list[Trace] = []
     if config.resume is not None:
         group = bool(discover_decorated(env.taskset, "group_reward"))
-        keep, owed = resume.plan(
+        finished, owed = resume.load(
             out, [t.idx for t in tasks], config.num_rollouts, group
         )
         if not owed:  # already complete - report it and exit successfully
             print(resume.nothing_to_resume_msg(out, len(tasks), config.num_rollouts))
             raise SystemExit(0)
         tasks = [task for task in tasks if owed.get(task.idx)]
-        resume.rewrite_results(out, keep)
-        if config.rich:
-            finished = resume.load_kept(out, env.taskset)
         logger.info(
             "resuming %s: %d task(s), %d rollout(s) owed",
             out,
@@ -88,7 +84,9 @@ async def run_eval(env: Environment, config: EvalConfig) -> list[Trace]:
             env.episode(task, ctx, n=owed[task.idx] if owed else config.num_rollouts)
             for task in tasks
         ]
-        rollouts = [rollout for episode in episodes for rollout in episode.rollouts]
+        rollouts = [resume.Finished(trace) for trace in finished] + [
+            rollout for episode in episodes for rollout in episode.rollouts
+        ]
         # --push in the live dashboard: share a status the dashboard renders as a line under the
         # rollouts once the run finishes and the upload begins (dim -> green URL / red error), and
         # do the upload inline below so it resolves on screen. Only the rich path has a dashboard;
@@ -99,7 +97,7 @@ async def run_eval(env: Environment, config: EvalConfig) -> list[Trace]:
 
             push_state = PushState()
         display = (
-            dashboard(rollouts, config, start, finished=finished, push=push_state)
+            dashboard(rollouts, config, start, push=push_state)
             if config.rich
             else contextlib.nullcontext()
         )
@@ -107,7 +105,9 @@ async def run_eval(env: Environment, config: EvalConfig) -> list[Trace]:
             results = await asyncio.gather(
                 *(episode.run(semaphore, on_complete) for episode in episodes)
             )
-            traces = [trace for episode_traces in results for trace in episode_traces]
+            traces = finished + [
+                trace for episode_traces in results for trace in episode_traces
+            ]
             if (
                 push_state is not None
             ):  # upload off the event loop so the view keeps refreshing
@@ -178,14 +178,14 @@ async def run_eval_server(config: EvalConfig) -> list[Trace]:
         if config.num_tasks is not None:
             idxs = idxs[: config.num_tasks]
         out = output_path(config)
+        finished: list[Trace] = []
         if config.resume is not None:
-            keep, owed = resume.plan(
+            finished, owed = resume.load(
                 out, idxs, config.num_rollouts, info.requires_group_scoring
             )
             if not owed:  # already complete - report it and exit successfully
                 print(resume.nothing_to_resume_msg(out, len(idxs), config.num_rollouts))
                 raise SystemExit(0)
-            resume.rewrite_results(out, keep)
             idxs = [idx for idx in idxs if owed.get(idx)]
             logger.info(
                 "resuming %s: %d task(s), %d rollout(s) owed",
@@ -248,7 +248,7 @@ async def run_eval_server(config: EvalConfig) -> list[Trace]:
             units = [run_rollout_unit(i) for i in idxs for _ in range(owed[i])]
         results = await asyncio.gather(*units)
         await client.close()
-        return [trace for unit_traces in results for trace in unit_traces]
+        return finished + [trace for unit_traces in results for trace in unit_traces]
     finally:
         proc.terminate()
         with contextlib.suppress(Exception):

verifiers-0.1.15.dev441/verifiers/v1/cli/eval/resume.py

@@ -1,138 +0,0 @@
-"""Resume an interrupted eval: re-run only the rollouts a previous run didn't finish.
-
-A run writes `config.toml` + `traces.jsonl` into its output dir. `--resume <dir>` reloads
-that config verbatim (so it takes no other flags) and writes back into the same dir, running
-only the rollouts still owed: the *missing* ones (never written — the run was interrupted) and
-the *errored* ones (written with an error). Good rollouts are kept; errored ones are dropped
-and redone. A group-scored taskset is resumed a whole task at a time (its rollouts are scored
-together), so any task that isn't fully complete is redone from scratch.
-"""
-
-import json
-import tomllib
-from collections import defaultdict
-from collections.abc import Iterator
-from pathlib import Path
-
-from pydantic_core import from_json
-
-from verifiers.v1.cli.output import CONFIG_FILE, TRACES_FILE, read_traces
-from verifiers.v1.configs.eval import EvalConfig
-from verifiers.v1.state import state_cls
-from verifiers.v1.task import WireTask
-from verifiers.v1.taskset import Taskset
-from verifiers.v1.trace import Trace
-
-
-def split_resume(argv: list[str]) -> tuple[Path | None, list[str]]:
-    """Pull `--resume <dir>` / `--resume=<dir>` out of argv, returning (dir, the other args).
-    The caller rejects any leftover args, since resume re-runs the saved config verbatim."""
-    for i, arg in enumerate(argv):
-        if arg == "--resume":
-            if i + 1 >= len(argv):
-                raise SystemExit(
-                    "--resume needs an output dir: uv run eval --resume <dir>"
-                )
-            return Path(argv[i + 1]), argv[:i] + argv[i + 2 :]
-        if arg.startswith("--resume="):
-            return Path(arg.split("=", 1)[1]), argv[:i] + argv[i + 1 :]
-    return None, argv
-
-
-def load_resume_config(resume_dir: Path) -> EvalConfig:
-    """Rebuild the run's `EvalConfig` from its saved `config.toml`, pointed back at its own
-    output dir so the resumed rollouts append to the same `traces.jsonl`."""
-    config_path = resume_dir / CONFIG_FILE
-    if not config_path.exists():
-        raise SystemExit(
-            f"--resume: no config.toml in {resume_dir} - not an eval output dir"
-        )
-    config = EvalConfig.model_validate(tomllib.loads(config_path.read_text()))
-    config.resume = resume_dir
-    config.output_dir = resume_dir
-    return config
-
-
-def _read_results(results_path: Path) -> Iterator[tuple[int, int, bool]]:
-    """Stream `(file reference, task idx, errored)` without retaining decoded traces."""
-    if not results_path.exists():
-        return
-    with results_path.open("rb") as results:
-        while True:
-            offset = results.tell()
-            line = results.readline()
-            if not line:
-                break
-            if line.strip():
-                try:
-                    row = from_json(line)
-                except ValueError:
-                    row = json.loads(line)
-                yield offset, row["task"]["idx"], bool(row.get("errors"))
-
-
-def plan(
-    resume_dir: Path, selected_idxs: list[int], num_rollouts: int, group: bool
-) -> tuple[list[int], dict[int, int]]:
-    """Diff the saved results against the run's target (`num_rollouts` per selected task).
-    Returns (byte offsets of rows to keep, rollouts owed per task idx). An errored trace is
-    dropped and re-run; a group-scored task is kept only if fully complete, else its whole group
-    is redone."""
-    # Retain only the offsets resume can reuse; trace payloads stay on disk.
-    selected = set(selected_idxs)
-    by_idx: dict[int, list[int]] = defaultdict(list)
-    for offset, idx, errored in _read_results(resume_dir / TRACES_FILE):
-        if idx in selected and not errored and len(by_idx[idx]) < num_rollouts:
-            by_idx[idx].append(offset)
-    keep: list[int] = []
-    owed: dict[int, int] = {}
-    for idx in selected_idxs:
-        good = by_idx.get(idx, [])
-        if group:
-            if len(good) >= num_rollouts:
-                keep.extend(good)
-            else:
-                owed[idx] = num_rollouts  # re-run the whole group; keep none of it
-        else:
-            keep.extend(good)
-            missing = num_rollouts - len(good)
-            if missing:
-                owed[idx] = missing
-    return keep, owed
-
-
-def rewrite_results(resume_dir: Path, keep: list[int]) -> None:
-    """Replace `traces.jsonl` with just the kept (good) traces; resumed rollouts append. Via a
-    temp file + atomic rename, so an interrupted resume can't corrupt the prior good results."""
-    path = resume_dir / TRACES_FILE
-    tmp = path.with_suffix(".jsonl.tmp")
-    if not keep:
-        tmp.write_bytes(b"")
-        tmp.replace(path)
-        return
-    # Re-read retained rows by offset while building the atomic replacement.
-    with path.open("rb") as results, tmp.open("wb") as output:
-        for offset in keep:
-            results.seek(offset)
-            raw = results.readline()
-            output.write(raw)
-            if not raw.endswith(b"\n"):
-                output.write(b"\n")
-    tmp.replace(path)
-
-
-def load_kept(resume_dir: Path, taskset: Taskset) -> list[Trace]:
-    """Reload the kept (good) traces as finished `Trace`s, so a resumed run's live dashboard counts
-    them toward the whole run (progress, reward, err, and the usage/time breakdown). Call *after*
-    `rewrite_results`, which leaves only the kept rows on disk. `WireTask` reads any taskset's saved
-    task without a runtime or its `Task` type (mirrors `replay`)."""
-    return read_traces(resume_dir, Trace[WireTask, state_cls(type(taskset))])
-
-
-def nothing_to_resume_msg(resume_dir: Path, num_tasks: int, num_rollouts: int) -> str:
-    """The message shown (and then exit 0 - the run is already complete) when every selected
-    rollout already completed without error."""
-    return (
-        f"nothing to resume in {resume_dir}: all {num_tasks}x{num_rollouts} rollouts "
-        f"already completed without error"
-    )