verifiers 0.1.15.dev424__tar.gz → 0.1.15.dev426__tar.gz

{verifiers-0.1.15.dev424 → verifiers-0.1.15.dev426}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.1.15.dev424
+Version: 0.1.15.dev426
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.1.15.dev424 → verifiers-0.1.15.dev426}/pyproject.toml

@@ -167,6 +167,7 @@ vf-eval = "verifiers.scripts.eval:main"
 eval = "verifiers.v1.cli.eval.main:main"
 validate = "verifiers.v1.cli.validate:main"
 debug = "verifiers.v1.cli.debug:main"
+replay = "verifiers.v1.cli.replay:main"
 serve = "verifiers.v1.cli.serve:main"
 init = "verifiers.v1.cli.init:main"
 vf-gepa = "verifiers.scripts.gepa:main"

verifiers-0.1.15.dev426/verifiers/v1/cli/dashboard/replay.py

@@ -0,0 +1,96 @@
+"""Live dashboard for `replay` — one row per trace being re-scored (pending → running → its
+outcome), mirroring the validate view. Reuses `TaskProgress` and `live_view`."""
+
+import contextlib
+import time
+from dataclasses import dataclass
+
+from rich.console import Group
+from rich.markup import escape
+from rich.progress_bar import ProgressBar
+from rich.rule import Rule
+from rich.table import Table
+from rich.text import Text
+
+from verifiers.v1.cli.dashboard.base import live_view
+from verifiers.v1.cli.dashboard.validate import TaskProgress
+from verifiers.v1.utils.format import format_time
+
+
+@dataclass
+class ReplayProgress(TaskProgress):
+    """`TaskProgress` plus the re-score `detail` shown per row: the final reward when scored, or
+    the exception type when it errored."""
+
+    detail: str = ""
+
+
+# Colors mirror the eval dashboard for shared states: pending=dim, running=cyan, scored=green
+# (eval's "success"), error=red. `skipped` is replay-only (a rollout that errored → copied through).
+_STYLE = {
+    "pending": "dim",
+    "running": "cyan",
+    "scored": "green",
+    "skipped": "blue",
+    "error": "red",
+}
+_MARK_WIDTH = max(len(state) for state in _STYLE)
+_MARK = {state: escape(f"[{state:<{_MARK_WIDTH}}]") for state in _STYLE}
+_DONE = ("scored", "error", "skipped")
+
+
+def _render(
+    states: list[TaskProgress], taskset_name: str, source: str, out: str, start: float
+) -> Group:
+    done = [s for s in states if s.state in _DONE]
+    scored = sum(1 for s in done if s.state == "scored")
+    skipped = sum(1 for s in done if s.state == "skipped")
+    overview = Table.grid(padding=(0, 2))
+    overview.add_column(style="dim")
+    overview.add_column()
+    overview.add_row("replay", f"{taskset_name}  ·  {source}")
+    overview.add_row("output", out)
+
+    stats = (
+        f" {len(done)}/{len(states)} · {format_time(time.time() - start)} · "
+        f"scored {scored} · failed {len(done) - scored - skipped} · skipped {skipped}"
+    )
+    progress = Table.grid()
+    progress.add_column()
+    progress.add_column()
+    progress.add_row(
+        ProgressBar(total=len(states) or 1, completed=len(done), width=32), Text(stats)
+    )
+
+    now = time.time()
+    rows = Table.grid(expand=True, padding=(0, 1))
+    rows.add_column(ratio=1, no_wrap=True)
+    rows.add_column(justify="right", no_wrap=True)
+    rows.add_column(justify="right", no_wrap=True)
+    for s in states:
+        if s.state == "pending":  # show only in-flight/done rows (like eval/validate)
+            continue
+        label = f"name={s.name[:40]}" if s.name else f"idx={s.idx}"
+        parts = [
+            p
+            for p in (s.state if s.state in _DONE else "", getattr(s, "detail", ""))
+            if p
+        ]
+        result = " ".join(parts) + " ·" if parts else ""
+        elapsed = format_time((s.end or now) - s.start) if s.start else ""
+        rows.add_row(
+            f"{_MARK[s.state]} {label}",
+            result,
+            elapsed,
+            style=_STYLE[s.state],
+        )
+    return Group(overview, progress, Rule(style="dim"), rows)
+
+
+@contextlib.asynccontextmanager
+async def replay_dashboard(
+    states: list[TaskProgress], taskset_name: str, source: str, out: str, start: float
+):
+    """Refresh the live replay view until the `with` block exits, then a final frame."""
+    async with live_view(lambda: _render(states, taskset_name, source, out, start)):
+        yield

{verifiers-0.1.15.dev424 → verifiers-0.1.15.dev426}/verifiers/v1/cli/output.py

@@ -12,6 +12,7 @@ durable as they land rather than only at the end.
 """
 
 import asyncio
+import json
 from pathlib import Path
 
 import tomli_w
@@ -58,6 +59,20 @@ def write_trace(results_dir: Path, trace: Trace) -> None:
         f.write(data + b"\n")
 
 
+def read_traces(results_dir: Path, trace_type: type) -> list[Trace]:
+    """Load a run's saved traces from `results.jsonl`, typed as `trace_type` — the inverse of
+    `write_trace`. Used by `replay` to re-score finished rollouts (pass the taskset's typed
+    `Trace[...]`, or `Trace[WireTask, ...]` to read any taskset's traces without importing it).
+    Streams line-by-line so a large (multi-GB) results file isn't loaded into memory at once."""
+    adapter = TypeAdapter(trace_type)
+    traces: list[Trace] = []
+    with (results_dir / "results.jsonl").open(encoding="utf-8") as f:
+        for line in f:
+            if line.strip():
+                traces.append(adapter.validate_python(json.loads(line)))
+    return traces
+
+
 async def append_trace(results_dir: Path, trace: Trace, lock: asyncio.Lock) -> None:
     """Append one finished trace without blocking the event loop. The run's shared lock
     preserves whole-line ordering, and awaiting the worker preserves per-trace durability."""

verifiers-0.1.15.dev426/verifiers/v1/cli/replay.py

@@ -0,0 +1,183 @@
+"""The replay entrypoint: `uv run replay <output-dir> [options] [@ file.toml]`.
+
+Offline sibling of `eval`: loads a finished run's saved traces and re-runs **scoring only**, no
+runtime. The run's own `config.toml` is the base; CLI flags / `@ file.toml` layer on top (e.g. a
+different judge model). Signals needing a `runtime` (in-sandbox verifiers like a SWE `solved`
+reward) are skipped and left as the source recorded them; config-plugged judges and trace-only
+`@reward`/`@metric`s re-run. Rollouts that errored during generation (`stop_condition == "error"`)
+were never scored by eval, so they're copied through unchanged rather than re-scored on a broken
+transcript. `--num-rescores`/`-r` re-scores each trace N times to sample judge variance. Results go
+to a fresh output dir, so the source run is never overwritten.
+"""
+
+import asyncio
+import contextlib
+import logging
+import signal
+import sys
+import time
+import tomllib
+from pathlib import Path
+
+from pydantic_config import cli
+
+import verifiers.v1 as vf
+from verifiers.v1.cli.dashboard.replay import ReplayProgress, replay_dashboard
+from verifiers.v1.cli.output import append_trace, read_traces, save_config, write_config
+from verifiers.v1.configs.replay import ReplayConfig
+from verifiers.v1.state import state_cls
+from verifiers.v1.task import WireTask
+from verifiers.v1.trace import Trace
+from verifiers.v1.utils.logging import setup_logging
+
+logger = logging.getLogger(__name__)
+
+USAGE = (
+    "usage: uv run replay <output-dir> [options] [@ file.toml]\n"
+    "       re-score a finished run's saved traces (judges + trace-only signals; no runtime)"
+)
+
+
+def _narrow(config_path: Path) -> type[ReplayConfig]:
+    """`ReplayConfig` with `taskset` narrowed to the saved run's taskset type, so the `cli()`
+    parse stays typed and CLI overrides of taskset fields validate (mirrors eval/validate)."""
+    data = tomllib.loads(config_path.read_text())
+    taskset_id = (data.get("taskset") or {}).get("id")
+    if not taskset_id:
+        return ReplayConfig
+    ftype = vf.taskset_config_type(taskset_id)
+    return type(
+        "ReplayConfig",
+        (ReplayConfig,),
+        {"__annotations__": {"taskset": ftype}, "taskset": ftype(id=taskset_id)},
+    )
+
+
+def output_dir(config: ReplayConfig) -> Path:
+    """Where a replay writes: `--output-dir`, else a fresh `outputs/<taskset>--replay/<uuid>`."""
+    return config.output_dir or Path("outputs") / f"{config.name}--replay" / config.uuid
+
+
+async def run_replay(config: ReplayConfig, source: Path, out: Path) -> list[Trace]:
+    logger.debug("replay config:\n%s", config.model_dump_json(indent=2))
+    taskset = vf.load_taskset(config.taskset)
+    # `WireTask` reads any taskset's saved task without a runtime or its Task type (see WireTask).
+    traces = read_traces(source, Trace[WireTask, state_cls(type(taskset))])
+    if config.num_traces is not None:
+        traces = traces[: config.num_traces]
+    # `num_rescores` re-scores each trace that many times, each on its own copy.
+    work = [t.model_copy(deep=True) for t in traces for _ in range(config.num_rescores)]
+
+    save_config(config, out)
+    logger.info(
+        "replay: re-scoring %d trace(s) x%d from %s -> %s",
+        len(traces),
+        config.num_rescores,
+        source,
+        out,
+    )
+    start = time.time()
+
+    sem = asyncio.Semaphore(config.max_concurrent) if config.max_concurrent else None
+    states = [ReplayProgress(idx=t.task.idx, name=t.task.name) for t in work]
+    lock = asyncio.Lock()
+
+    async def rescore(trace: Trace, st: ReplayProgress) -> None:
+        async with sem or contextlib.nullcontext():
+            st.start = time.time()
+            # A rollout that errored during generation was never scored by eval (its scoring phase
+            # never ran), so re-scoring would grade a broken/partial transcript. Copy it through
+            # unchanged, mirroring eval. A *scoring* error leaves `stop_condition` as the
+            # generation outcome, so those still re-score (the offline-recovery case).
+            if trace.stop_condition == "error":
+                st.state, st.detail, st.end = "skipped", "rollout errored", time.time()
+            else:
+                st.state = "running"
+                # Clear prior scores so a changed/removed judge leaves no stale (double-summed) entry.
+                if isinstance(trace.info, dict):
+                    trace.info.pop("judge", None)
+                trace.rewards, trace.metrics, trace.extra_usage = {}, {}, []
+                try:
+                    await taskset.score(trace)
+                    st.state, st.detail = "scored", f"reward {trace.reward:.3f}"
+                except Exception as exc:
+                    st.state, st.detail = "error", type(exc).__name__
+                    trace.capture_error(
+                        exc
+                    )  # record the re-scoring failure on the trace
+                    if not config.rich:
+                        logger.warning(
+                            "replay: scoring failed for task %s",
+                            trace.task.idx,
+                            exc_info=True,
+                        )
+                st.end = time.time()
+        if not config.rich:
+            logger.info(
+                "idx=%s %s (%.1fs)", trace.task.idx, st.state, st.end - st.start
+            )
+        # Persist as each trace finishes (like eval), so an interrupted replay keeps its progress.
+        await append_trace(out, trace, lock)
+
+    display = (
+        replay_dashboard(states, config.name, str(source), str(out), start)
+        if config.rich
+        else contextlib.nullcontext()
+    )
+    async with display:
+        await asyncio.gather(*(rescore(t, s) for t, s in zip(work, states)))
+    logger.info("replay: done in %.1fs -> %s", time.time() - start, out)
+    return work
+
+
+def main(argv: list[str] | None = None) -> None:
+    argv = list(sys.argv[1:]) if argv is None else list(argv)
+    if not argv or any(a in ("-h", "--help") for a in argv):
+        print(USAGE)
+        sys.argv = [sys.argv[0], "--help"]
+        cli(ReplayConfig)  # full, typed pydantic-config option help
+        return
+    source = Path(argv.pop(0))  # the finished run dir to replay
+    config_path = source / "config.toml"
+    if not config_path.exists():
+        raise SystemExit(f"{USAGE}\nno config.toml in {source}")
+
+    # The saved run's config is the base; user flags / @ file.toml layer on top.
+    layered = ["@", str(config_path), *argv]
+    config_type = _narrow(config_path)
+    sys.argv = [sys.argv[0], *layered]
+    config = cli(config_type)
+    # Honor a user-set output_dir (via -o or an @ file); drop the *source* run's own output_dir
+    # so a replay always writes to a fresh dir and never back over the run it re-scores.
+    source_out = (tomllib.loads(config_path.read_text())).get("output_dir")
+    if config.output_dir is None or str(config.output_dir) == str(source_out):
+        config = config.model_copy(update={"output_dir": None})
+
+    out = output_dir(config)
+    if (
+        out.resolve() == source.resolve()
+    ):  # never write back over the run being replayed
+        raise SystemExit(
+            f"replay: --output-dir must differ from the source run ({source}); "
+            "refusing to overwrite it"
+        )
+    level = "DEBUG" if config.verbose else "INFO"
+    if config.dry_run:  # resolve + validate, write the config, and exit (no re-scoring)
+        setup_logging(level)
+        logger.info("wrote config to %s", write_config(config, out))
+        return
+
+    log_file = str(
+        out / "replay.log"
+    )  # tee the run's logs to its output dir, like eval
+    if config.rich:  # the dashboard owns the screen, so keep logs off the console
+        setup_logging(level, log_file=log_file, console=False)
+        logging.lastResort = None
+    else:
+        setup_logging(level, log_file=log_file, console=True)
+    signal.signal(signal.SIGTERM, lambda *_: (_ for _ in ()).throw(KeyboardInterrupt()))
+    asyncio.run(run_replay(config, source, out))
+
+
+if __name__ == "__main__":
+    main()

{verifiers-0.1.15.dev424 → verifiers-0.1.15.dev426}/verifiers/v1/clients/client.py

@@ -67,7 +67,13 @@ class Client(ABC):
         generates and cannot stream."""
         raise NotImplementedError(f"{type(self).__name__} does not support streaming")
 
-    async def relay_aux(self, dialect: Dialect, route: str, body: dict) -> dict:
+    async def relay_aux(
+        self,
+        dialect: Dialect,
+        route: str,
+        body: dict,
+        headers: Mapping[str, str] | None = None,
+    ) -> dict:
         """Relay a non-model-turn side request (an `aux_route`, e.g. Anthropic's `count_tokens`)
         verbatim to the provider and return its JSON. Only the relay (eval) client supports it."""
         raise NotImplementedError(f"{type(self).__name__} does not relay aux routes")

{verifiers-0.1.15.dev424 → verifiers-0.1.15.dev426}/verifiers/v1/clients/eval.py

@@ -205,12 +205,18 @@ class EvalClient(Client):
             close=resp.aclose,
         )
 
-    async def relay_aux(self, dialect: Dialect, route: str, body: dict) -> dict:
+    async def relay_aux(
+        self,
+        dialect: Dialect,
+        route: str,
+        body: dict,
+        headers: Mapping[str, str] | None = None,
+    ) -> dict:
         # A side request (e.g. count_tokens): relay its native JSON and return the provider JSON.
         resp = await self._request(
             self.base_url + route,
             body,
-            self._headers(dialect, None, None),
+            self._headers(dialect, headers, None),
         )
         return from_json(resp.content)
 

verifiers-0.1.15.dev426/verifiers/v1/configs/replay.py

@@ -0,0 +1,67 @@
+"""The `ReplayConfig`: the config the `replay` CLI parses.
+
+Replay re-scores a finished run's saved traces with no runtime, so it needs only a subset of
+`EvalConfig` — the taskset (with its plugged judges) and a few run knobs. It carries no harness,
+runtime, model, sampling, or pool. Its base is the replayed run's own `config.toml` (a full
+`EvalConfig`), so `model_config` ignores the eval-only fields rather than rejecting them; CLI
+flags / `@ file.toml` then layer overrides (e.g. re-judge with different judge settings).
+"""
+
+from pathlib import Path
+from uuid import uuid4
+
+from pydantic import AliasChoices, ConfigDict, Field, SerializeAsAny, model_validator
+from pydantic_config import BaseConfig
+
+from verifiers.v1.taskset import TasksetConfig
+
+
+class ReplayConfig(BaseConfig):
+    """A taskset (with its config-plugged judges) plus replay run knobs. The saved run's
+    `config.toml` is the base; its eval-only fields (harness, runtime, model, sampling, pool, …)
+    are ignored here — replay only re-runs scoring from the trace."""
+
+    model_config = ConfigDict(extra="ignore")
+
+    uuid: str = Field(default_factory=lambda: str(uuid4()), exclude=True)
+    """Auto-generated run id — the leaf of the output dir, so replays never overwrite."""
+    taskset: SerializeAsAny[TasksetConfig] = TasksetConfig()
+    """The taskset, selected by `--taskset.id` (narrowed to its config type). Its
+    `taskset.judges` are what replay re-runs; override them via `@ file.toml` / dotted flags."""
+    num_traces: int | None = Field(
+        None, validation_alias=AliasChoices("num_traces", "n")
+    )
+    """How many saved traces to re-score (None = all)."""
+    num_rescores: int = Field(1, validation_alias=AliasChoices("num_rescores", "r"))
+    """Re-score each selected trace this many times — e.g. to measure judge variance across
+    repeated gradings of the same trace. Named distinctly from eval's `num_rollouts` so the
+    replayed run's own `num_rollouts` (in the base `config.toml`) is ignored, not inherited."""
+    max_concurrent: int | None = Field(
+        128, validation_alias=AliasChoices("max_concurrent", "c")
+    )
+    """Max traces re-scored (judge calls) in flight at once."""
+    verbose: bool = Field(False, validation_alias=AliasChoices("verbose", "v"))
+    """Log at debug level instead of the default info."""
+    dry_run: bool = False
+    """Resolve + validate the config and write it to the output dir, then exit (no re-scoring)."""
+    rich: bool = True
+    """Show a live dashboard (one row per trace) instead of per-trace log lines."""
+    output_dir: Path | None = Field(
+        None, validation_alias=AliasChoices("output_dir", "o")
+    )
+    """Where to write the re-scored run (config.toml + results.jsonl). None = a fresh per-run
+    dir under `outputs/<taskset>--replay/<uuid>` (so a replay never overwrites the source run)."""
+
+    @property
+    def name(self) -> str:
+        return self.taskset.name
+
+    @model_validator(mode="before")
+    @classmethod
+    def _resolve_taskset(cls, data):
+        """Narrow the generic `taskset` to its specific config type by `id` (mirrors
+        `ValidateConfig`), so `taskset.judges` and other taskset fields validate typed."""
+        from verifiers.v1.loaders import narrow_plugin_field, taskset_config_type
+
+        narrow_plugin_field(data, "taskset", taskset_config_type)
+        return data

{verifiers-0.1.15.dev424 → verifiers-0.1.15.dev426}/verifiers/v1/interception/server.py

@@ -543,7 +543,7 @@ class InterceptionServer:
         logger.debug("intercept aux %s: id=%s", route, session.trace.id)
         try:
             result = await session.ctx.client.relay_aux(
-                dialect, route, await request.json()
+                dialect, route, await request.json(), headers=request.headers
             )
         except RolloutError as e:
             # An aux call isn't a model turn, so don't clobber a pending turn error.

{verifiers-0.1.15.dev424 → verifiers-0.1.15.dev426}/verifiers/v1/taskset.py

@@ -17,6 +17,8 @@ For a heterogeneous taskset (different verification per task), have a single
 """
 
 import asyncio
+import inspect
+import logging
 from collections.abc import Mapping
 from functools import cached_property
 from typing import ClassVar, Generic, TypeVar
@@ -35,6 +37,8 @@ from verifiers.v1.state import StateT
 from verifiers.v1.task import TaskT
 from verifiers.v1.trace import Trace
 
+logger = logging.getLogger(__name__)
+
 
 class TasksetConfig(BaseConfig):
     """Base taskset config. Subclass to add task-generation knobs."""
@@ -172,17 +176,36 @@ class Taskset(Generic[TaskT, ConfigT, StateT]):
         records the reason (the raised error's message)."""
         return True
 
-    async def score(self, trace: Trace, runtime: Runtime) -> None:
+    async def score(self, trace: Trace, runtime: Runtime | None = None) -> None:
         """Score one rollout: run all `@metric`, then `@reward`, then the config-plugged
         `judges` over its trace, concurrently within each phase. Each metric is recorded in
         `trace.metrics` (a number, or a mapping merged in); each reward and judge verdict
         (weighted — likewise a number or a mapping merged in) in `trace.rewards`, which
         `trace.reward` sums. Signals declare what they need — `task`, `trace`,
         `runtime` — so a reward is either a pure function of the trace or runs
-        read/write/exec in that (still-live) runtime, e.g. a verifier script."""
-        available = {"task": trace.task, "trace": trace, "runtime": runtime}
+        read/write/exec in that (still-live) runtime, e.g. a verifier script.
+
+        `runtime` may be `None` — the offline path taken by `replay`, which re-scores a saved
+        trace with no live runtime. Signals that declare a `runtime` parameter are then skipped
+        (they can't run without one) and logged; trace-only `@metric`/`@reward`s and the plugged
+        judges (which grade from the trace) still run, overwriting their own keys and leaving the
+        skipped signals' previously-recorded values untouched. With a runtime present (the eval
+        path) nothing is skipped."""
+        available = {"task": trace.task, "trace": trace}
+        if runtime is not None:
+            available["runtime"] = runtime
+
+        def can_run(fn) -> bool:
+            # A signal that *requires* a runtime can't run without one, so skip it when replaying.
+            # A `runtime` parameter with a default (e.g. `runtime=None`) is optional — `invoke`
+            # just omits it — so those still run offline.
+            if runtime is not None:
+                return True
+            param = inspect.signature(fn).parameters.get("runtime")
+            return param is None or param.default is not inspect.Parameter.empty
+
         async with boundary(TasksetError, f"taskset {type(self).__name__} scoring"):
-            metrics = discover_decorated(self, "metric")
+            metrics = [fn for fn in discover_decorated(self, "metric") if can_run(fn)]
             metric_results = (
                 [await invoke(fn, available) for fn in metrics]
                 if len(metrics) < 2
@@ -193,7 +216,7 @@ class Taskset(Generic[TaskT, ConfigT, StateT]):
                     trace.record_metrics(result)
                 else:
                     trace.record_metric(fn.__name__, result)
-            rewards = discover_decorated(self, "reward")
+            rewards = [fn for fn in discover_decorated(self, "reward") if can_run(fn)]
             reward_results = (
                 [await invoke(fn, available) for fn in rewards]
                 if len(rewards) < 2
@@ -206,7 +229,24 @@ class Taskset(Generic[TaskT, ConfigT, StateT]):
                         trace.record_reward(name, value, weight)
                 else:
                     trace.record_reward(fn.__name__, result, weight)
-            judges = self.judges
+            judges = [judge for judge in self.judges if can_run(judge.score)]
+            if runtime is None:
+                # Exactly the signals `can_run` filtered out above — those requiring a runtime
+                # (a `runtime` param with no default). Signals whose `runtime` has a default run
+                # offline and are not reported as skipped.
+                skipped = [
+                    fn.__name__
+                    for fn in (
+                        *discover_decorated(self, "metric"),
+                        *discover_decorated(self, "reward"),
+                    )
+                    if not can_run(fn)
+                ] + [j.reward_name for j in self.judges if not can_run(j.score)]
+                if skipped:
+                    logger.info(
+                        "score: no runtime — skipped runtime-dependent signals: %s",
+                        skipped,
+                    )
             judge_results = (
                 [await invoke(judge.score, available) for judge in judges]
                 if len(judges) < 2