verifiers 0.1.15.dev436__tar.gz → 0.1.15.dev438__tar.gz

{verifiers-0.1.15.dev436 → verifiers-0.1.15.dev438}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.1.15.dev436
+Version: 0.1.15.dev438
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.1.15.dev436 → verifiers-0.1.15.dev438}/verifiers/v1/ARCHITECTURE.md

@@ -69,7 +69,7 @@ several. A branch *is* a training sample: concatenating its nodes' `token_ids` r
 end to end: each surviving context window is just another root→leaf path.
 
 `Trace.to_record()` (`trace.py`) is the JSON record dump (`model_dump(mode="json")`) for
-`results.jsonl` / W&B tables, minus the per-node training tensors (`MessageNode.multi_modal_data`,
+`traces.jsonl` / W&B tables, minus the per-node training tensors (`MessageNode.multi_modal_data`,
 `routed_experts`, via `_NODE_DUMP_EXCLUDE`): those hold raw numpy bytes that can't round-trip JSON
 (the dump raises `UnicodeDecodeError` on real expert ids) and bloat every line. Computed views
 (`reward`, `branches`, `num_turns`, per-span `duration`) are pydantic properties, so they're never

{verifiers-0.1.15.dev436 → verifiers-0.1.15.dev438}/verifiers/v1/GUIDE.md

@@ -369,7 +369,7 @@ Good to know:
 - **What lands on the trace:** the weighted verdict in `trace.rewards[<reward key>]`; rubric's
   raw per-criterion verdicts in `trace.metrics`; every call's `JudgeResponse` in
   `trace.info["judge"]` with its tokens + cost in `trace.extra_usage` (shown as `+judge` in the
-  dashboard). All persisted to `results.jsonl`.
+  dashboard). All persisted to `traces.jsonl`.
 
 Writing your own pluggable judge is the same recipe as any plugin: a package exporting a `Judge`
 subclass via `__all__` that implements `score` — declare any subset of `task` / `trace` / `runtime`
@@ -468,7 +468,7 @@ class SWETaskset(vf.Taskset[SWETask, SWEConfig]):
 `trace.info` is a free-form, **JSON-serializable** dict for per-rollout artifacts that are neither a
 reward nor a metric — the diff above, captured logs, command output, file paths. Write to it from
 `finalize` or a `@reward`/`@metric` by assigning into the dict; it is persisted with the trace
-(dumped to `results.jsonl` and sent over the wire), so every value must be JSON-serializable — a
+(dumped to `traces.jsonl` and sent over the wire), so every value must be JSON-serializable — a
 non-serializable value fails the trace dump rather than being silently dropped.
 
 ```python
@@ -815,7 +815,7 @@ client-side so each node carries the exact `token_ids` / `mask` / `logprobs` (th
 — point it at a vLLM engine with `--client.base-url`.
 
 **What it writes** (into `--output-dir`, default `outputs/<taskset>--<model>--<harness>/<uuid>`):
-`config.toml` (the resolved config, re-runnable via `@ config.toml`), `results.jsonl` (one full
+`config.toml` (the resolved config, re-runnable via `@ config.toml`), `traces.jsonl` (one full
 trace per line, appended as each rollout finishes — durable mid-run), and `eval.log`.
 
 **`--dry-run`** writes `config.toml` and exits (resolve + validate, no run). **`--resume
@@ -869,7 +869,7 @@ uv run debug swebench-v1 -n 1 --runtime.type prime --script-path ./inspect.sh
 | `--runtime.type` | `docker` | runtime for setup + the debug action |
 | `--timeout.setup` / `--timeout.total` | None | per-task wall-clock caps for `setup` and the debug action |
 | `-n`/`--num-tasks`, `-s`/`--shuffle`, `-c`/`--max-concurrent` (128) | | task selection + concurrency |
-| `-o`/`--output-dir` | fresh debug run dir | where `config.toml` and `results.jsonl` are written |
+| `-o`/`--output-dir` | fresh debug run dir | where `config.toml` and `traces.jsonl` are written |
 
 Each saved trace has command/script metadata, exit status, elapsed time, timeout/error fields,
 and the full stdout/stderr under `trace.info["debug"]`.

{verifiers-0.1.15.dev436 → verifiers-0.1.15.dev438}/verifiers/v1/cli/eval/resume.py

@@ -1,6 +1,6 @@
 """Resume an interrupted eval: re-run only the rollouts a previous run didn't finish.
 
-A run writes `config.toml` + `results.jsonl` into its output dir. `--resume <dir>` reloads
+A run writes `config.toml` + `traces.jsonl` into its output dir. `--resume <dir>` reloads
 that config verbatim (so it takes no other flags) and writes back into the same dir, running
 only the rollouts still owed: the *missing* ones (never written — the run was interrupted) and
 the *errored* ones (written with an error). Good rollouts are kept; errored ones are dropped
@@ -16,7 +16,7 @@ from pathlib import Path
 
 from pydantic_core import from_json
 
-from verifiers.v1.cli.output import read_traces
+from verifiers.v1.cli.output import CONFIG_FILE, TRACES_FILE, read_traces
 from verifiers.v1.configs.eval import EvalConfig
 from verifiers.v1.state import state_cls
 from verifiers.v1.task import WireTask
@@ -41,8 +41,8 @@ def split_resume(argv: list[str]) -> tuple[Path | None, list[str]]:
 
 def load_resume_config(resume_dir: Path) -> EvalConfig:
     """Rebuild the run's `EvalConfig` from its saved `config.toml`, pointed back at its own
-    output dir so the resumed rollouts append to the same `results.jsonl`."""
-    config_path = resume_dir / "config.toml"
+    output dir so the resumed rollouts append to the same `traces.jsonl`."""
+    config_path = resume_dir / CONFIG_FILE
     if not config_path.exists():
         raise SystemExit(
             f"--resume: no config.toml in {resume_dir} - not an eval output dir"
@@ -81,7 +81,7 @@ def plan(
     # Retain only the offsets resume can reuse; trace payloads stay on disk.
     selected = set(selected_idxs)
     by_idx: dict[int, list[int]] = defaultdict(list)
-    for offset, idx, errored in _read_results(resume_dir / "results.jsonl"):
+    for offset, idx, errored in _read_results(resume_dir / TRACES_FILE):
         if idx in selected and not errored and len(by_idx[idx]) < num_rollouts:
             by_idx[idx].append(offset)
     keep: list[int] = []
@@ -102,9 +102,9 @@ def plan(
 
 
 def rewrite_results(resume_dir: Path, keep: list[int]) -> None:
-    """Replace `results.jsonl` with just the kept (good) traces; resumed rollouts append. Via a
+    """Replace `traces.jsonl` with just the kept (good) traces; resumed rollouts append. Via a
     temp file + atomic rename, so an interrupted resume can't corrupt the prior good results."""
-    path = resume_dir / "results.jsonl"
+    path = resume_dir / TRACES_FILE
     tmp = path.with_suffix(".jsonl.tmp")
     if not keep:
         tmp.write_bytes(b"")

{verifiers-0.1.15.dev436 → verifiers-0.1.15.dev438}/verifiers/v1/cli/eval/runner.py

@@ -123,7 +123,7 @@ async def run_eval_server(config: EvalConfig) -> list[Trace]:
     """Eval through the env-server worker pool (`--num-workers > 0`). Spawns the pool
     (works for v1 and the v0 bridge), then drives rollouts by task idx over an
     `EnvClient` — the same path prime-rl trains through, so it exercises the
-    router + workers end-to-end. Output matches `run_eval` (config.toml + results.jsonl)."""
+    router + workers end-to-end. Output matches `run_eval` (config.toml + traces.jsonl)."""
     import multiprocessing as mp
     from functools import partial
 

{verifiers-0.1.15.dev436 → verifiers-0.1.15.dev438}/verifiers/v1/cli/output.py

@@ -1,4 +1,4 @@
-"""On-disk output: results.jsonl (one full trace per line) + config.toml.
+"""On-disk output: traces.jsonl (one full trace per line) + config.toml.
 
 The trace is the full data dump — written verbatim, consumed by the platform
 (visualization) and prime-rl (training). config.toml is the run's resolved EvalConfig,
@@ -7,7 +7,7 @@ its own output. Aggregates (avg reward, etc.) are cheap to recompute from result
 they aren't stored.
 
 The runner writes `config.toml` once up front (`save_config`) and then appends each
-trace to `results.jsonl` as it completes (`append_trace`), so a long run's results are
+trace to `traces.jsonl` as it completes (`append_trace`), so a long run's results are
 durable as they land rather than only at the end.
 """
 
@@ -22,6 +22,12 @@ from verifiers.v1.configs.eval import EvalConfig
 from verifiers.v1.trace import Trace
 from verifiers.v1.utils.aio import run_shielded
 
+TRACES_FILE = "traces.jsonl"
+"""Filename each run's full per-rollout traces are written to (one JSON trace per line)."""
+
+CONFIG_FILE = "config.toml"
+"""Filename a run's resolved config is written to (re-runnable via `@ config.toml`)."""
+
 
 def output_path(config: EvalConfig) -> Path:
     """Where this run writes: `outputs/<taskset>--<model>--<harness>/<uuid>` (or the explicit
@@ -38,35 +44,33 @@ def write_config(config: BaseModel, results_dir: Path) -> Path:
     nulls TOML can't represent."""
     results_dir.mkdir(parents=True, exist_ok=True)
     toml = tomli_w.dumps(config.model_dump(mode="json", exclude_none=True))
-    config_path = results_dir / "config.toml"
+    config_path = results_dir / CONFIG_FILE
     config_path.write_text(toml)
     return config_path
 
 
 def save_config(config: BaseModel, results_dir: Path) -> None:
     """Set up the run's output dir: write `config.toml` and start a fresh (empty)
-    `results.jsonl`. Call once up front, before traces start landing."""
+    `traces.jsonl`. Call once up front, before traces start landing."""
     write_config(config, results_dir)
-    (results_dir / "results.jsonl").write_text(
-        ""
-    )  # fresh; appended to as traces complete
+    (results_dir / TRACES_FILE).write_text("")  # fresh; appended to as traces complete
 
 
 def write_trace(results_dir: Path, trace: Trace) -> None:
     """Serialize and append one trace in the worker thread."""
     data = TypeAdapter(type(trace)).dump_json(trace, exclude_none=True)
-    with (results_dir / "results.jsonl").open("ab") as f:
+    with (results_dir / TRACES_FILE).open("ab") as f:
         f.write(data + b"\n")
 
 
 def read_traces(results_dir: Path, trace_type: type) -> list[Trace]:
-    """Load a run's saved traces from `results.jsonl`, typed as `trace_type` — the inverse of
+    """Load a run's saved traces from `traces.jsonl`, typed as `trace_type` — the inverse of
     `write_trace`. Used by `replay` to re-score finished rollouts (pass the taskset's typed
     `Trace[...]`, or `Trace[WireTask, ...]` to read any taskset's traces without importing it).
-    Streams line-by-line so a large (multi-GB) results file isn't loaded into memory at once."""
+    Streams line-by-line so a large (multi-GB) traces file isn't loaded into memory at once."""
     adapter = TypeAdapter(trace_type)
     traces: list[Trace] = []
-    with (results_dir / "results.jsonl").open(encoding="utf-8") as f:
+    with (results_dir / TRACES_FILE).open(encoding="utf-8") as f:
         for line in f:
             if line.strip():
                 traces.append(adapter.validate_python(json.loads(line)))

{verifiers-0.1.15.dev436 → verifiers-0.1.15.dev438}/verifiers/v1/cli/replay.py

@@ -23,7 +23,13 @@ from pydantic_config import cli
 
 import verifiers.v1 as vf
 from verifiers.v1.cli.dashboard.replay import ReplayProgress, replay_dashboard
-from verifiers.v1.cli.output import append_trace, read_traces, save_config, write_config
+from verifiers.v1.cli.output import (
+    CONFIG_FILE,
+    append_trace,
+    read_traces,
+    save_config,
+    write_config,
+)
 from verifiers.v1.configs.replay import ReplayConfig
 from verifiers.v1.state import state_cls
 from verifiers.v1.task import WireTask
@@ -138,7 +144,7 @@ def main(argv: list[str] | None = None) -> None:
         cli(ReplayConfig)  # full, typed pydantic-config option help
         return
     source = Path(argv.pop(0))  # the finished run dir to replay
-    config_path = source / "config.toml"
+    config_path = source / CONFIG_FILE
     if not config_path.exists():
         raise SystemExit(f"{USAGE}\nno config.toml in {source}")
 

{verifiers-0.1.15.dev436 → verifiers-0.1.15.dev438}/verifiers/v1/configs/debug.py

@@ -46,7 +46,7 @@ class DebugConfig(BaseConfig):
     output_dir: Path | None = Field(
         None, validation_alias=AliasChoices("output_dir", "o")
     )
-    """Where to write `config.toml` and `results.jsonl`. None = a fresh per-run dir."""
+    """Where to write `config.toml` and `traces.jsonl`. None = a fresh per-run dir."""
 
     @property
     def name(self) -> str:

{verifiers-0.1.15.dev436 → verifiers-0.1.15.dev438}/verifiers/v1/configs/eval.py

@@ -61,7 +61,7 @@ class EvalConfig(EnvServerConfig):
     output_dir: Path | None = Field(
         None, validation_alias=AliasChoices("output_dir", "o")
     )
-    """Where to write the run (config.toml + results.jsonl). None = a fresh per-run dir
+    """Where to write the run (config.toml + traces.jsonl). None = a fresh per-run dir
     under `outputs/<env>--<model>--<harness>/<uuid>` (so runs never overwrite each other)."""
     resume: Path | None = Field(None, exclude=True)
     """Set by `--resume <dir>`: re-run only the rollouts a previous run left missing or

{verifiers-0.1.15.dev436 → verifiers-0.1.15.dev438}/verifiers/v1/configs/replay.py

@@ -49,7 +49,7 @@ class ReplayConfig(BaseConfig):
     output_dir: Path | None = Field(
         None, validation_alias=AliasChoices("output_dir", "o")
     )
-    """Where to write the re-scored run (config.toml + results.jsonl). None = a fresh per-run
+    """Where to write the re-scored run (config.toml + traces.jsonl). None = a fresh per-run
     dir under `outputs/<taskset>--replay/<uuid>` (so a replay never overwrites the source run)."""
 
     @property

{verifiers-0.1.15.dev436 → verifiers-0.1.15.dev438}/verifiers/v1/harnesses/codex/harness.py

@@ -105,6 +105,7 @@ class CodexHarness(Harness[CodexHarnessConfig]):
             "-c",
             f"model_providers.{PROVIDER}.requires_openai_auth=false",
             *tool_config,
+            "--",
             prompt,
         ]
         return await runtime.run_program(argv, env)

{verifiers-0.1.15.dev436 → verifiers-0.1.15.dev438}/verifiers/v1/harnesses/rlm/harness.py

@@ -140,7 +140,7 @@ class RLMHarness(Harness[RLMHarnessConfig]):
             env["RLM_MCP_CONFIG"] = json.dumps(
                 {"mcpServers": {name: {"url": url} for name, url in mcp_urls.items()}}
             )
-        return await runtime.run_program([RLM_BIN, prompt], env)
+        return await runtime.run_program([RLM_BIN, "--", prompt], env)
 
     @metric
     async def rlm(self, trace: Trace, runtime: Runtime) -> dict[str, float]:

{verifiers-0.1.15.dev436 → verifiers-0.1.15.dev438}/verifiers/v1/legacy.py

@@ -435,7 +435,7 @@ async def run_legacy_eval(config) -> list[Trace]:
 
     Loads the v0 env, runs `num_rollouts` per task with bounded concurrency, maps each v0
     `RolloutOutput` to a v1 `Trace` (`rollout_output_to_trace`), persists results as they
-    land (the same `results.jsonl` / `config.toml` a native run writes), and returns the
+    land (the same `traces.jsonl` / `config.toml` a native run writes), and returns the
     traces. The v0 env is run directly (`env.run_rollout`, no env server), so this needs no
     runtime / interception server. All v0 specifics live here; the CLI only branches on
     `config.is_legacy`."""

{verifiers-0.1.15.dev436 → verifiers-0.1.15.dev438}/verifiers/v1/state.py

@@ -3,7 +3,7 @@
 `Trace.state` is a typed, mutable `State` that a rollout's tool servers (`@vf.tool`) and user
 simulator (`respond`) read+write as `self.state` (synced over the interception server per call), and
 that `@reward`/`@metric`/`finalize` read+write directly off the trace. Unlike `Trace.info` — the
-free-form artifact bag persisted to `results.jsonl` — `state` is transient runtime scratch (counters,
+free-form artifact bag persisted to `traces.jsonl` — `state` is transient runtime scratch (counters,
 game progress, an end-of-trajectory flag): never written to disk or sent over the wire.
 
 The base `State` is empty — the framework holds no opinion about its contents. Subclass it to declare

{verifiers-0.1.15.dev436 → verifiers-0.1.15.dev438}/verifiers/v1/trace.py

@@ -2,7 +2,7 @@
 
 A `Trace[TaskT]` carries the typed task plus everything produced during a
 rollout (conversation, per-turn responses, reward, metrics, timing, error). It is
-the canonical full data dump — written to disk (`results.jsonl`) and consumed by
+the canonical full data dump — written to disk (`traces.jsonl`) and consumed by
 the platform (visualization) and prime-rl (training). Environments subclass it to
 add typed scratch/result fields. The rollout mutates it directly; this replaces
 v1's 600-line `dict`-subclass `State` and its dual "contract version" machinery.
@@ -234,7 +234,7 @@ class Trace(StrictBaseModel, Generic[TaskT, StateT]):
     a reward nor a metric — anything an author wants to scrape off the live runtime and persist
     with the trace (captured logs, command output, container/runtime state, artifact paths).
     Populate it from the runtime in `finalize` (or a `@reward`/`@metric`) by assigning into the
-    dict (`trace.info["build_log"] = ...`); it round-trips through the wire to `results.jsonl`.
+    dict (`trace.info["build_log"] = ...`); it round-trips through the wire to `traces.jsonl`.
     Use `metrics` for numbers that aggregate, this for everything else. Values must be
     JSON-serializable — a non-serializable value fails the trace dump rather than being dropped."""
     state: StateT = Field(default_factory=State, exclude=True)
@@ -464,7 +464,7 @@ class Trace(StrictBaseModel, Generic[TaskT, StateT]):
         self.stop("error")
 
     def to_record(self) -> dict[str, Any]:
-        """A JSON-serializable record of this rollout for `results.jsonl` / W&B tables.
+        """A JSON-serializable record of this rollout for `traces.jsonl` / W&B tables.
 
         `model_dump(mode="json")` minus the per-node training tensors (`_NODE_DUMP_EXCLUDE`):
         those carry raw numpy bytes that JSON can't encode (the plain dump raises