verifiers 0.1.15.dev423__tar.gz → 0.1.15.dev425__tar.gz

{verifiers-0.1.15.dev423 → verifiers-0.1.15.dev425}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.1.15.dev423
+Version: 0.1.15.dev425
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.1.15.dev423 → verifiers-0.1.15.dev425}/pyproject.toml

@@ -167,6 +167,7 @@ vf-eval = "verifiers.scripts.eval:main"
 eval = "verifiers.v1.cli.eval.main:main"
 validate = "verifiers.v1.cli.validate:main"
 debug = "verifiers.v1.cli.debug:main"
+replay = "verifiers.v1.cli.replay:main"
 serve = "verifiers.v1.cli.serve:main"
 init = "verifiers.v1.cli.init:main"
 vf-gepa = "verifiers.scripts.gepa:main"

{verifiers-0.1.15.dev423 → verifiers-0.1.15.dev425}/verifiers/v1/__init__.py

@@ -12,7 +12,7 @@ from verifiers.v1.clients import (
     BaseClientConfig,
     Client,
     ClientConfig,
-    RolloutContext,
+    ModelContext,
     resolve_client,
 )
 from verifiers.v1.decorators import group_reward, metric, reward, stop, tool
@@ -188,7 +188,7 @@ __all__ = [
     "BaseConfig",
     "Harness",
     "HarnessConfig",
-    "RolloutContext",
+    "ModelContext",
     "Runtime",
     "RuntimeConfig",
     "ProgramResult",

verifiers-0.1.15.dev425/verifiers/v1/cli/dashboard/replay.py

@@ -0,0 +1,96 @@
+"""Live dashboard for `replay` — one row per trace being re-scored (pending → running → its
+outcome), mirroring the validate view. Reuses `TaskProgress` and `live_view`."""
+
+import contextlib
+import time
+from dataclasses import dataclass
+
+from rich.console import Group
+from rich.markup import escape
+from rich.progress_bar import ProgressBar
+from rich.rule import Rule
+from rich.table import Table
+from rich.text import Text
+
+from verifiers.v1.cli.dashboard.base import live_view
+from verifiers.v1.cli.dashboard.validate import TaskProgress
+from verifiers.v1.utils.format import format_time
+
+
+@dataclass
+class ReplayProgress(TaskProgress):
+    """`TaskProgress` plus the re-score `detail` shown per row: the final reward when scored, or
+    the exception type when it errored."""
+
+    detail: str = ""
+
+
+# Colors mirror the eval dashboard for shared states: pending=dim, running=cyan, scored=green
+# (eval's "success"), error=red. `skipped` is replay-only (a rollout that errored → copied through).
+_STYLE = {
+    "pending": "dim",
+    "running": "cyan",
+    "scored": "green",
+    "skipped": "blue",
+    "error": "red",
+}
+_MARK_WIDTH = max(len(state) for state in _STYLE)
+_MARK = {state: escape(f"[{state:<{_MARK_WIDTH}}]") for state in _STYLE}
+_DONE = ("scored", "error", "skipped")
+
+
+def _render(
+    states: list[TaskProgress], taskset_name: str, source: str, out: str, start: float
+) -> Group:
+    done = [s for s in states if s.state in _DONE]
+    scored = sum(1 for s in done if s.state == "scored")
+    skipped = sum(1 for s in done if s.state == "skipped")
+    overview = Table.grid(padding=(0, 2))
+    overview.add_column(style="dim")
+    overview.add_column()
+    overview.add_row("replay", f"{taskset_name}  ·  {source}")
+    overview.add_row("output", out)
+
+    stats = (
+        f" {len(done)}/{len(states)} · {format_time(time.time() - start)} · "
+        f"scored {scored} · failed {len(done) - scored - skipped} · skipped {skipped}"
+    )
+    progress = Table.grid()
+    progress.add_column()
+    progress.add_column()
+    progress.add_row(
+        ProgressBar(total=len(states) or 1, completed=len(done), width=32), Text(stats)
+    )
+
+    now = time.time()
+    rows = Table.grid(expand=True, padding=(0, 1))
+    rows.add_column(ratio=1, no_wrap=True)
+    rows.add_column(justify="right", no_wrap=True)
+    rows.add_column(justify="right", no_wrap=True)
+    for s in states:
+        if s.state == "pending":  # show only in-flight/done rows (like eval/validate)
+            continue
+        label = f"name={s.name[:40]}" if s.name else f"idx={s.idx}"
+        parts = [
+            p
+            for p in (s.state if s.state in _DONE else "", getattr(s, "detail", ""))
+            if p
+        ]
+        result = " ".join(parts) + " ·" if parts else ""
+        elapsed = format_time((s.end or now) - s.start) if s.start else ""
+        rows.add_row(
+            f"{_MARK[s.state]} {label}",
+            result,
+            elapsed,
+            style=_STYLE[s.state],
+        )
+    return Group(overview, progress, Rule(style="dim"), rows)
+
+
+@contextlib.asynccontextmanager
+async def replay_dashboard(
+    states: list[TaskProgress], taskset_name: str, source: str, out: str, start: float
+):
+    """Refresh the live replay view until the `with` block exits, then a final frame."""
+    async with live_view(lambda: _render(states, taskset_name, source, out, start)):
+        yield

{verifiers-0.1.15.dev423 → verifiers-0.1.15.dev425}/verifiers/v1/cli/eval/runner.py

@@ -6,7 +6,7 @@ import logging
 import random
 import time
 
-from verifiers.v1.clients import RolloutContext, resolve_client
+from verifiers.v1.clients import ModelContext, resolve_client
 from verifiers.v1.configs.eval import EvalConfig
 from verifiers.v1.cli.eval import resume
 from verifiers.v1.cli.dashboard import dashboard
@@ -29,7 +29,7 @@ async def run_eval(env: Environment, config: EvalConfig) -> list[Trace]:
     if config.shuffle:
         random.Random(_SHUFFLE_SEED).shuffle(tasks)
     tasks = tasks if config.num_tasks is None else tasks[: config.num_tasks]
-    ctx = RolloutContext(client=client, model=config.model, sampling=config.sampling)
+    ctx = ModelContext(client=client, model=config.model, sampling=config.sampling)
     # One episode of `num_rollouts` rollouts per task; the shared semaphore bounds total
     # concurrent rollouts (across episodes), so group rewards still see their whole episode.
     semaphore = (

{verifiers-0.1.15.dev423 → verifiers-0.1.15.dev425}/verifiers/v1/cli/init.py

@@ -192,7 +192,7 @@ class {prefix}Harness(vf.Harness[{prefix}HarnessConfig]):
 
     async def launch(
         self,
-        ctx: vf.RolloutContext,
+        ctx: vf.ModelContext,
         trace: vf.Trace,
         runtime: vf.Runtime,
         endpoint: str,

{verifiers-0.1.15.dev423 → verifiers-0.1.15.dev425}/verifiers/v1/cli/output.py

@@ -12,6 +12,7 @@ durable as they land rather than only at the end.
 """
 
 import asyncio
+import json
 from pathlib import Path
 
 import tomli_w
@@ -58,6 +59,20 @@ def write_trace(results_dir: Path, trace: Trace) -> None:
         f.write(data + b"\n")
 
 
+def read_traces(results_dir: Path, trace_type: type) -> list[Trace]:
+    """Load a run's saved traces from `results.jsonl`, typed as `trace_type` — the inverse of
+    `write_trace`. Used by `replay` to re-score finished rollouts (pass the taskset's typed
+    `Trace[...]`, or `Trace[WireTask, ...]` to read any taskset's traces without importing it).
+    Streams line-by-line so a large (multi-GB) results file isn't loaded into memory at once."""
+    adapter = TypeAdapter(trace_type)
+    traces: list[Trace] = []
+    with (results_dir / "results.jsonl").open(encoding="utf-8") as f:
+        for line in f:
+            if line.strip():
+                traces.append(adapter.validate_python(json.loads(line)))
+    return traces
+
+
 async def append_trace(results_dir: Path, trace: Trace, lock: asyncio.Lock) -> None:
     """Append one finished trace without blocking the event loop. The run's shared lock
     preserves whole-line ordering, and awaiting the worker preserves per-trace durability."""

verifiers-0.1.15.dev425/verifiers/v1/cli/replay.py

@@ -0,0 +1,183 @@
+"""The replay entrypoint: `uv run replay <output-dir> [options] [@ file.toml]`.
+
+Offline sibling of `eval`: loads a finished run's saved traces and re-runs **scoring only**, no
+runtime. The run's own `config.toml` is the base; CLI flags / `@ file.toml` layer on top (e.g. a
+different judge model). Signals needing a `runtime` (in-sandbox verifiers like a SWE `solved`
+reward) are skipped and left as the source recorded them; config-plugged judges and trace-only
+`@reward`/`@metric`s re-run. Rollouts that errored during generation (`stop_condition == "error"`)
+were never scored by eval, so they're copied through unchanged rather than re-scored on a broken
+transcript. `--num-rescores`/`-r` re-scores each trace N times to sample judge variance. Results go
+to a fresh output dir, so the source run is never overwritten.
+"""
+
+import asyncio
+import contextlib
+import logging
+import signal
+import sys
+import time
+import tomllib
+from pathlib import Path
+
+from pydantic_config import cli
+
+import verifiers.v1 as vf
+from verifiers.v1.cli.dashboard.replay import ReplayProgress, replay_dashboard
+from verifiers.v1.cli.output import append_trace, read_traces, save_config, write_config
+from verifiers.v1.configs.replay import ReplayConfig
+from verifiers.v1.state import state_cls
+from verifiers.v1.task import WireTask
+from verifiers.v1.trace import Trace
+from verifiers.v1.utils.logging import setup_logging
+
+logger = logging.getLogger(__name__)
+
+USAGE = (
+    "usage: uv run replay <output-dir> [options] [@ file.toml]\n"
+    "       re-score a finished run's saved traces (judges + trace-only signals; no runtime)"
+)
+
+
+def _narrow(config_path: Path) -> type[ReplayConfig]:
+    """`ReplayConfig` with `taskset` narrowed to the saved run's taskset type, so the `cli()`
+    parse stays typed and CLI overrides of taskset fields validate (mirrors eval/validate)."""
+    data = tomllib.loads(config_path.read_text())
+    taskset_id = (data.get("taskset") or {}).get("id")
+    if not taskset_id:
+        return ReplayConfig
+    ftype = vf.taskset_config_type(taskset_id)
+    return type(
+        "ReplayConfig",
+        (ReplayConfig,),
+        {"__annotations__": {"taskset": ftype}, "taskset": ftype(id=taskset_id)},
+    )
+
+
+def output_dir(config: ReplayConfig) -> Path:
+    """Where a replay writes: `--output-dir`, else a fresh `outputs/<taskset>--replay/<uuid>`."""
+    return config.output_dir or Path("outputs") / f"{config.name}--replay" / config.uuid
+
+
+async def run_replay(config: ReplayConfig, source: Path, out: Path) -> list[Trace]:
+    logger.debug("replay config:\n%s", config.model_dump_json(indent=2))
+    taskset = vf.load_taskset(config.taskset)
+    # `WireTask` reads any taskset's saved task without a runtime or its Task type (see WireTask).
+    traces = read_traces(source, Trace[WireTask, state_cls(type(taskset))])
+    if config.num_traces is not None:
+        traces = traces[: config.num_traces]
+    # `num_rescores` re-scores each trace that many times, each on its own copy.
+    work = [t.model_copy(deep=True) for t in traces for _ in range(config.num_rescores)]
+
+    save_config(config, out)
+    logger.info(
+        "replay: re-scoring %d trace(s) x%d from %s -> %s",
+        len(traces),
+        config.num_rescores,
+        source,
+        out,
+    )
+    start = time.time()
+
+    sem = asyncio.Semaphore(config.max_concurrent) if config.max_concurrent else None
+    states = [ReplayProgress(idx=t.task.idx, name=t.task.name) for t in work]
+    lock = asyncio.Lock()
+
+    async def rescore(trace: Trace, st: ReplayProgress) -> None:
+        async with sem or contextlib.nullcontext():
+            st.start = time.time()
+            # A rollout that errored during generation was never scored by eval (its scoring phase
+            # never ran), so re-scoring would grade a broken/partial transcript. Copy it through
+            # unchanged, mirroring eval. A *scoring* error leaves `stop_condition` as the
+            # generation outcome, so those still re-score (the offline-recovery case).
+            if trace.stop_condition == "error":
+                st.state, st.detail, st.end = "skipped", "rollout errored", time.time()
+            else:
+                st.state = "running"
+                # Clear prior scores so a changed/removed judge leaves no stale (double-summed) entry.
+                if isinstance(trace.info, dict):
+                    trace.info.pop("judge", None)
+                trace.rewards, trace.metrics, trace.extra_usage = {}, {}, []
+                try:
+                    await taskset.score(trace)
+                    st.state, st.detail = "scored", f"reward {trace.reward:.3f}"
+                except Exception as exc:
+                    st.state, st.detail = "error", type(exc).__name__
+                    trace.capture_error(
+                        exc
+                    )  # record the re-scoring failure on the trace
+                    if not config.rich:
+                        logger.warning(
+                            "replay: scoring failed for task %s",
+                            trace.task.idx,
+                            exc_info=True,
+                        )
+                st.end = time.time()
+        if not config.rich:
+            logger.info(
+                "idx=%s %s (%.1fs)", trace.task.idx, st.state, st.end - st.start
+            )
+        # Persist as each trace finishes (like eval), so an interrupted replay keeps its progress.
+        await append_trace(out, trace, lock)
+
+    display = (
+        replay_dashboard(states, config.name, str(source), str(out), start)
+        if config.rich
+        else contextlib.nullcontext()
+    )
+    async with display:
+        await asyncio.gather(*(rescore(t, s) for t, s in zip(work, states)))
+    logger.info("replay: done in %.1fs -> %s", time.time() - start, out)
+    return work
+
+
+def main(argv: list[str] | None = None) -> None:
+    argv = list(sys.argv[1:]) if argv is None else list(argv)
+    if not argv or any(a in ("-h", "--help") for a in argv):
+        print(USAGE)
+        sys.argv = [sys.argv[0], "--help"]
+        cli(ReplayConfig)  # full, typed pydantic-config option help
+        return
+    source = Path(argv.pop(0))  # the finished run dir to replay
+    config_path = source / "config.toml"
+    if not config_path.exists():
+        raise SystemExit(f"{USAGE}\nno config.toml in {source}")
+
+    # The saved run's config is the base; user flags / @ file.toml layer on top.
+    layered = ["@", str(config_path), *argv]
+    config_type = _narrow(config_path)
+    sys.argv = [sys.argv[0], *layered]
+    config = cli(config_type)
+    # Honor a user-set output_dir (via -o or an @ file); drop the *source* run's own output_dir
+    # so a replay always writes to a fresh dir and never back over the run it re-scores.
+    source_out = (tomllib.loads(config_path.read_text())).get("output_dir")
+    if config.output_dir is None or str(config.output_dir) == str(source_out):
+        config = config.model_copy(update={"output_dir": None})
+
+    out = output_dir(config)
+    if (
+        out.resolve() == source.resolve()
+    ):  # never write back over the run being replayed
+        raise SystemExit(
+            f"replay: --output-dir must differ from the source run ({source}); "
+            "refusing to overwrite it"
+        )
+    level = "DEBUG" if config.verbose else "INFO"
+    if config.dry_run:  # resolve + validate, write the config, and exit (no re-scoring)
+        setup_logging(level)
+        logger.info("wrote config to %s", write_config(config, out))
+        return
+
+    log_file = str(
+        out / "replay.log"
+    )  # tee the run's logs to its output dir, like eval
+    if config.rich:  # the dashboard owns the screen, so keep logs off the console
+        setup_logging(level, log_file=log_file, console=False)
+        logging.lastResort = None
+    else:
+        setup_logging(level, log_file=log_file, console=True)
+    signal.signal(signal.SIGTERM, lambda *_: (_ for _ in ()).throw(KeyboardInterrupt()))
+    asyncio.run(run_replay(config, source, out))
+
+
+if __name__ == "__main__":
+    main()

{verifiers-0.1.15.dev423 → verifiers-0.1.15.dev425}/verifiers/v1/clients/__init__.py

@@ -1,6 +1,6 @@
 """The client abstraction and its OpenAI-compatible + renderer implementations."""
 
-from verifiers.v1.clients.client import Client, RolloutContext
+from verifiers.v1.clients.client import Client, ModelContext
 from verifiers.v1.clients.config import (
     BaseClientConfig,
     ClientConfig,
@@ -13,7 +13,7 @@ from verifiers.v1.clients.train import TrainClient
 
 __all__ = [
     "Client",
-    "RolloutContext",
+    "ModelContext",
     "BaseClientConfig",
     "ClientConfig",
     "EvalClientConfig",

{verifiers-0.1.15.dev423 → verifiers-0.1.15.dev425}/verifiers/v1/clients/client.py

@@ -77,9 +77,9 @@ class Client(ABC):
 
 
 @dataclass(frozen=True)
-class RolloutContext:
-    """The collaborators a single rollout needs (client + model + sampling), bundled
-    so harnesses hold no rollout state. Built by the Environment."""
+class ModelContext:
+    """The model-side collaborators a harness talks to (client + model + sampling),
+    bundled so harnesses hold no rollout state. Built by the Environment."""
 
     model: str
     client: Client

verifiers-0.1.15.dev425/verifiers/v1/configs/replay.py

@@ -0,0 +1,67 @@
+"""The `ReplayConfig`: the config the `replay` CLI parses.
+
+Replay re-scores a finished run's saved traces with no runtime, so it needs only a subset of
+`EvalConfig` — the taskset (with its plugged judges) and a few run knobs. It carries no harness,
+runtime, model, sampling, or pool. Its base is the replayed run's own `config.toml` (a full
+`EvalConfig`), so `model_config` ignores the eval-only fields rather than rejecting them; CLI
+flags / `@ file.toml` then layer overrides (e.g. re-judge with different judge settings).
+"""
+
+from pathlib import Path
+from uuid import uuid4
+
+from pydantic import AliasChoices, ConfigDict, Field, SerializeAsAny, model_validator
+from pydantic_config import BaseConfig
+
+from verifiers.v1.taskset import TasksetConfig
+
+
+class ReplayConfig(BaseConfig):
+    """A taskset (with its config-plugged judges) plus replay run knobs. The saved run's
+    `config.toml` is the base; its eval-only fields (harness, runtime, model, sampling, pool, …)
+    are ignored here — replay only re-runs scoring from the trace."""
+
+    model_config = ConfigDict(extra="ignore")
+
+    uuid: str = Field(default_factory=lambda: str(uuid4()), exclude=True)
+    """Auto-generated run id — the leaf of the output dir, so replays never overwrite."""
+    taskset: SerializeAsAny[TasksetConfig] = TasksetConfig()
+    """The taskset, selected by `--taskset.id` (narrowed to its config type). Its
+    `taskset.judges` are what replay re-runs; override them via `@ file.toml` / dotted flags."""
+    num_traces: int | None = Field(
+        None, validation_alias=AliasChoices("num_traces", "n")
+    )
+    """How many saved traces to re-score (None = all)."""
+    num_rescores: int = Field(1, validation_alias=AliasChoices("num_rescores", "r"))
+    """Re-score each selected trace this many times — e.g. to measure judge variance across
+    repeated gradings of the same trace. Named distinctly from eval's `num_rollouts` so the
+    replayed run's own `num_rollouts` (in the base `config.toml`) is ignored, not inherited."""
+    max_concurrent: int | None = Field(
+        128, validation_alias=AliasChoices("max_concurrent", "c")
+    )
+    """Max traces re-scored (judge calls) in flight at once."""
+    verbose: bool = Field(False, validation_alias=AliasChoices("verbose", "v"))
+    """Log at debug level instead of the default info."""
+    dry_run: bool = False
+    """Resolve + validate the config and write it to the output dir, then exit (no re-scoring)."""
+    rich: bool = True
+    """Show a live dashboard (one row per trace) instead of per-trace log lines."""
+    output_dir: Path | None = Field(
+        None, validation_alias=AliasChoices("output_dir", "o")
+    )
+    """Where to write the re-scored run (config.toml + results.jsonl). None = a fresh per-run
+    dir under `outputs/<taskset>--replay/<uuid>` (so a replay never overwrites the source run)."""
+
+    @property
+    def name(self) -> str:
+        return self.taskset.name
+
+    @model_validator(mode="before")
+    @classmethod
+    def _resolve_taskset(cls, data):
+        """Narrow the generic `taskset` to its specific config type by `id` (mirrors
+        `ValidateConfig`), so `taskset.judges` and other taskset fields validate typed."""
+        from verifiers.v1.loaders import narrow_plugin_field, taskset_config_type
+
+        narrow_plugin_field(data, "taskset", taskset_config_type)
+        return data

{verifiers-0.1.15.dev423 → verifiers-0.1.15.dev425}/verifiers/v1/env.py

@@ -19,7 +19,7 @@ from pydantic import Field, SerializeAsAny, model_validator
 from pydantic_config import BaseConfig
 
 from verifiers.v1.harness import HarnessConfig
-from verifiers.v1.clients import RolloutContext
+from verifiers.v1.clients import ModelContext
 from verifiers.v1.decorators import discover_decorated
 from verifiers.v1.episode import Episode
 from verifiers.v1.types import ID
@@ -300,7 +300,7 @@ class Environment:
             self.harness.config.runtime, task, self._warned_resources
         )
 
-    def episode(self, task: Task, ctx: RolloutContext, n: int = 1) -> Episode:
+    def episode(self, task: Task, ctx: ModelContext, n: int = 1) -> Episode:
         """Resolve `task` into a runnable episode of `n` rollouts: pick its runtime
         (image + resources) and its timeouts (cli/toml > task > default, None = no limit),
         build one `Rollout` per sample sharing them, and wrap them in an `Episode` (which

{verifiers-0.1.15.dev423 → verifiers-0.1.15.dev425}/verifiers/v1/harness.py

@@ -16,7 +16,7 @@ from typing import ClassVar, Generic, TypeVar
 from pydantic import Field
 from pydantic_config import BaseConfig
 
-from verifiers.v1.clients import RolloutContext
+from verifiers.v1.clients import ModelContext
 from verifiers.v1.decorators import discover_decorated, invoke
 from verifiers.v1.errors import HarnessError, boundary
 from verifiers.v1.utils.install import env_name
@@ -131,7 +131,7 @@ class Harness(ABC, Generic[ConfigT]):
 
     async def run(
         self,
-        ctx: RolloutContext,
+        ctx: ModelContext,
         trace: Trace,
         runtime: Runtime,
         endpoint: str,
@@ -176,7 +176,7 @@ class Harness(ABC, Generic[ConfigT]):
     @abstractmethod
     async def launch(
         self,
-        ctx: RolloutContext,
+        ctx: ModelContext,
         trace: Trace,
         runtime: Runtime,
         endpoint: str,

{verifiers-0.1.15.dev423 → verifiers-0.1.15.dev425}/verifiers/v1/harnesses/codex/harness.py

@@ -10,7 +10,7 @@ session secret) is read from an env var.
 import logging
 import shlex
 
-from verifiers.v1.clients import RolloutContext
+from verifiers.v1.clients import ModelContext
 from verifiers.v1.harness import Harness, HarnessConfig
 from verifiers.v1.runtimes import ProgramResult, Runtime
 from verifiers.v1.trace import Trace
@@ -65,7 +65,7 @@ class CodexHarness(Harness[CodexHarnessConfig]):
 
     async def launch(
         self,
-        ctx: RolloutContext,
+        ctx: ModelContext,
         trace: Trace,
         runtime: Runtime,
         endpoint: str,

{verifiers-0.1.15.dev423 → verifiers-0.1.15.dev425}/verifiers/v1/harnesses/default/harness.py

@@ -14,7 +14,7 @@ import os
 from pathlib import Path
 
 from verifiers.v1.harness import Harness, HarnessConfig
-from verifiers.v1.clients import RolloutContext
+from verifiers.v1.clients import ModelContext
 from verifiers.v1.dialects.chat import message_to_wire
 from verifiers.v1.runtimes import ProgramResult, Runtime
 from verifiers.v1.trace import Trace
@@ -61,7 +61,7 @@ class DefaultHarness(Harness[DefaultHarnessConfig]):
 
     async def launch(
         self,
-        ctx: RolloutContext,
+        ctx: ModelContext,
         trace: Trace,
         runtime: Runtime,
         endpoint: str,

{verifiers-0.1.15.dev423 → verifiers-0.1.15.dev425}/verifiers/v1/harnesses/kimi_code/harness.py

@@ -9,7 +9,7 @@ import json
 import logging
 import shlex
 
-from verifiers.v1.clients import RolloutContext
+from verifiers.v1.clients import ModelContext
 from verifiers.v1.harness import Harness, HarnessConfig
 from verifiers.v1.runtimes import ProgramResult, Runtime
 from verifiers.v1.trace import Trace
@@ -64,7 +64,7 @@ class KimiCodeHarness(Harness[KimiCodeHarnessConfig]):
 
     async def launch(
         self,
-        ctx: RolloutContext,
+        ctx: ModelContext,
         trace: Trace,
         runtime: Runtime,
         endpoint: str,

{verifiers-0.1.15.dev423 → verifiers-0.1.15.dev425}/verifiers/v1/harnesses/mini_swe_agent/harness.py

@@ -2,7 +2,7 @@
 
 from pathlib import Path
 
-from verifiers.v1.clients import RolloutContext
+from verifiers.v1.clients import ModelContext
 from verifiers.v1.harness import Harness, HarnessConfig
 from verifiers.v1.runtimes import ProgramResult, Runtime
 from verifiers.v1.trace import Trace
@@ -27,7 +27,7 @@ class MiniSWEAgentHarness(Harness[MiniSWEAgentHarnessConfig]):
 
     async def launch(
         self,
-        ctx: RolloutContext,
+        ctx: ModelContext,
         trace: Trace,
         runtime: Runtime,
         endpoint: str,

{verifiers-0.1.15.dev423 → verifiers-0.1.15.dev425}/verifiers/v1/harnesses/null/harness.py

@@ -11,7 +11,7 @@ import json
 from pathlib import Path
 
 from verifiers.v1.harness import Harness, HarnessConfig
-from verifiers.v1.clients import RolloutContext
+from verifiers.v1.clients import ModelContext
 from verifiers.v1.dialects.chat import message_to_wire
 from verifiers.v1.runtimes import ProgramResult, Runtime
 from verifiers.v1.trace import Trace
@@ -35,7 +35,7 @@ class NullHarness(Harness[NullHarnessConfig]):
 
     async def launch(
         self,
-        ctx: RolloutContext,
+        ctx: ModelContext,
         trace: Trace,
         runtime: Runtime,
         endpoint: str,

{verifiers-0.1.15.dev423 → verifiers-0.1.15.dev425}/verifiers/v1/harnesses/rlm/harness.py

@@ -18,7 +18,7 @@ from typing import Literal
 from pydantic import model_validator
 
 from verifiers.v1.harness import Harness, HarnessConfig
-from verifiers.v1.clients import RolloutContext
+from verifiers.v1.clients import ModelContext
 from verifiers.v1.decorators import metric
 from verifiers.v1.runtimes import ProgramResult, Runtime
 from verifiers.v1.trace import Trace
@@ -115,7 +115,7 @@ class RLMHarness(Harness[RLMHarnessConfig]):
 
     async def launch(
         self,
-        ctx: RolloutContext,
+        ctx: ModelContext,
         trace: Trace,
         runtime: Runtime,
         endpoint: str,

{verifiers-0.1.15.dev423 → verifiers-0.1.15.dev425}/verifiers/v1/harnesses/terminus_2/harness.py

@@ -3,7 +3,7 @@
 import logging
 from pathlib import Path
 
-from verifiers.v1.clients import RolloutContext
+from verifiers.v1.clients import ModelContext
 from verifiers.v1.harness import Harness, HarnessConfig
 from verifiers.v1.runtimes import ProgramResult, Runtime
 from verifiers.v1.trace import Trace
@@ -39,7 +39,7 @@ class Terminus2Harness(Harness[Terminus2HarnessConfig]):
 
     async def launch(
         self,
-        ctx: RolloutContext,
+        ctx: ModelContext,
         trace: Trace,
         runtime: Runtime,
         endpoint: str,