verifiers 0.3.1.dev33__tar.gz → 0.3.1.dev35__tar.gz

{verifiers-0.3.1.dev33 → verifiers-0.3.1.dev35}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.3.1.dev33
+Version: 0.3.1.dev35
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers
@@ -51,7 +51,7 @@ Provides-Extra: browser
 Requires-Dist: python-dotenv>=1.0.0; extra == 'browser'
 Requires-Dist: stagehand>=3.4.0; extra == 'browser'
 Provides-Extra: harbor
-Requires-Dist: harbor==0.20.0; (python_version >= '3.12') and extra == 'harbor'
+Requires-Dist: harbor==0.21.0; (python_full_version >= '3.12') and extra == 'harbor'
 Provides-Extra: modal
 Requires-Dist: modal>=1.4.0; extra == 'modal'
 Provides-Extra: nemo-gym

{verifiers-0.3.1.dev33 → verifiers-0.3.1.dev35}/pyproject.toml

@@ -109,7 +109,7 @@ browser = [
 ]
 harbor = [
     # Harbor currently publishes Python 3.12+ packages; keep core verifiers installable on 3.11.
-    "harbor==0.20.0; python_version >= '3.12'",
+    "harbor==0.21.0 ; python_full_version >= '3.12'",
 ]
 modal = [
     "modal>=1.4.0",
@@ -186,6 +186,7 @@ web-search-interception = { path = "environments/web_search_interception", edita
 # Bounded cutoffs for the explicitly requested tool upgrades.
 # Full UTC timestamps: bare dates resolve to local-tz midnight and churn the
 # lockfile across timezones.
+harbor = "2026-08-11T00:00:00Z"
 ruff = "2026-07-28T00:00:00Z"
 ty = "2026-07-28T00:00:00Z"
 # PrimeIntellect-published on PyPI (trusted publisher)

{verifiers-0.3.1.dev33 → verifiers-0.3.1.dev35}/tests/v1/conftest.py

@@ -184,7 +184,8 @@ def _eval_config(
             "reasoning_effort": reasoning_effort,
         },
         rich=False,
-        output_dir=output_dir,
+        output_dir=output_dir.parent,
+        run={"dir": output_dir.name},
         **({"serve": {"pool": pool}} if pool else {}),
         model=CI_MODEL,
     )

{verifiers-0.3.1.dev33 → verifiers-0.3.1.dev35}/tests/v1/test_e2e.py

@@ -662,10 +662,9 @@ async def test_replay_round_trip(run_v1, tmp_path):
     typed rebuild fails and the trace-only `@reward` silently stops running (the
     wire-narrowing regression). Trace-only rewards are deterministic given the transcript,
     so all three generations must agree."""
-    import tomllib
     from pathlib import Path
 
-    from verifiers.v1.cli.output import CONFIG_FILE
+    from verifiers.v1.cli.output import saved_config_path
     from verifiers.v1.cli.replay import run_replay
     from verifiers.v1.configs.cli.replay import ReplayConfig
 
@@ -683,7 +682,9 @@ async def test_replay_round_trip(run_v1, tmp_path):
     async def replay(source_dir: Path, out: Path):
         # The CLI's layering, minus the argv plumbing: the saved run's config is the base
         # (`ReplayConfig` ignores its eval-only keys), the source's output_dir is dropped.
-        data = tomllib.loads((source_dir / CONFIG_FILE).read_text())
+        import json
+
+        data = json.loads(saved_config_path(source_dir).read_text())
         data.pop("output_dir", None)
         config = ReplayConfig(**{**data, "rich": False})
         (trace,) = await run_replay(config, source_dir, out)

verifiers-0.3.1.dev35/verifiers/v1/cli/eval/main.py

@@ -0,0 +1,152 @@
+"""Eval CLI entrypoint."""
+
+import asyncio
+import json
+import logging
+import shutil
+import sys
+
+from pydantic_config import cli
+
+import verifiers.v1 as vf
+from verifiers.v1.cli.eval.runner import run_eval
+from verifiers.v1.cli.output import (
+    TRACES_FILE,
+    config_digest,
+    output_path,
+    saved_config_path,
+    write_config,
+)
+from verifiers.v1.cli.resolve import (
+    extract_id,
+    narrow_config,
+    plugin_errors,
+    references_config_file,
+    with_positional_taskset,
+)
+from verifiers.v1.configs.cli.eval import EvalConfig
+from verifiers.v1.utils.interrupt import install_interrupt
+from verifiers.v1.utils.logging import setup_logging
+
+logger = logging.getLogger(__name__)
+
+USAGE = (
+    "usage: uv run eval [<taskset-id>] [--env.id <id>] [options] [@ file.toml]\n"
+    "       uv run eval @ <run-dir>/configs/eval.json --resume   (re-run the run's missing/errored rollouts)"
+)
+
+
+def main(argv: list[str] | None = None) -> None:
+    argv = with_positional_taskset(list(sys.argv[1:]) if argv is None else list(argv))
+
+    if not argv or any(arg in ("-h", "--help") for arg in argv):
+        print(USAGE)
+        sys.argv = [sys.argv[0], "--help"]
+        with plugin_errors():
+            cli(
+                narrow_config(EvalConfig, argv)
+            )  # full option help, narrowed to the given ids
+        return
+    # An env-block flag (or a since-moved flat axis) skips the usage gate so the
+    # typed parse renders its did-you-mean instead of a bare usage line.
+    typed_axis = any(
+        a.startswith(("--env.", "--taskset.", "--harness.", "--serve.")) for a in argv
+    )
+    if (
+        not extract_id(argv, "env.taskset")
+        and not references_config_file(argv)
+        and not typed_axis
+    ):
+        raise SystemExit(
+            USAGE
+        )  # need a taskset (positional / --env.taskset.id) or a @ file.toml
+
+    with plugin_errors():
+        config_type = narrow_config(EvalConfig, argv)
+        sys.argv = [
+            sys.argv[0],
+            *argv,
+        ]  # let prime-pydantic-config render help/errors
+        config = cli(config_type)
+    # A named run directory is re-entered only by `--resume` or wiped by `--clean`: any
+    # other write into it — the dry-run config.toml included, which would clobber the
+    # config a resume typically re-runs — would overwrite the previous run.
+    run_path = output_path(config)
+    if config.clean and not config.resume and run_path.exists():
+        if not run_path.resolve().is_relative_to(config.output_dir.resolve()):
+            raise SystemExit("--clean requires run.dir to remain under output_dir")
+        shutil.rmtree(run_path)
+    traces_file = run_path / TRACES_FILE
+    if not config.resume and traces_file.exists() and traces_file.stat().st_size > 0:
+        raise SystemExit(
+            f"run directory {run_path} already contains results - append --resume to "
+            "re-run its missing/errored rollouts, overwrite it with --clean, or pick "
+            "another --run.name"
+        )
+    if config.resume:
+        # A resumed eval is only trustworthy under the exact config that produced the
+        # existing rollouts — anything else silently mixes incomparable results. Saved
+        # configs are full JSON dumps (nulls included), so the comparison is exact.
+        saved_path = saved_config_path(run_path)
+        if saved_path is None:
+            raise SystemExit(
+                f"--resume: no saved config under {run_path} - not a run dir"
+            )
+        with plugin_errors():
+            saved = config_type.model_validate(json.loads(saved_path.read_text()))
+        if config_digest(saved) != config_digest(config):
+            saved_dump = saved.model_dump(mode="json")
+            current_dump = config.model_dump(mode="json")
+            changed = sorted(
+                key
+                for key in set(saved_dump) | set(current_dump)
+                if saved_dump.get(key) != current_dump.get(key)
+            )
+            raise SystemExit(
+                f"--resume requires the exact config the run was started with - it "
+                f"differs in [{', '.join(changed)}]. Resumed rollouts would not be "
+                f"comparable; re-run with `uv run eval @ {saved_path} --resume`, or "
+                "start a fresh run"
+            )
+    if config.dry_run:  # resolved + validated; write it to the output dir and exit
+        setup_logging("DEBUG" if config.verbose else "INFO")
+        logger.info("wrote config to %s", write_config(config, output_path(config)))
+        return
+    # Execution path: in-process by default; `--server` opts into the env-server worker pool
+    # (the path prime-rl trains through). The `--rich` dashboard reads live in-process run
+    # slots, so it's in-process only (`server + rich` is rejected at config validation).
+    # Always tee the run's logs to a file under the output dir (in-process and server mode).
+    log_file = str(output_path(config) / "logs" / "eval.log")
+    level = "DEBUG" if config.verbose else "INFO"
+    if config.rich:
+        setup_logging(level, log_file=log_file, console=False)
+        # drop stray stdlib records that bypass loguru (else they print over the UI)
+        logging.lastResort = None
+    else:
+        setup_logging(level, log_file=log_file, console=True)
+    # First Ctrl-C / SIGTERM warns and raises KeyboardInterrupt so a killed/timed-out eval still
+    # runs each rollout's `finally` (tears down containers/sandboxes) and any worker pool it
+    # spawned; further signals during that cleanup are swallowed so an impatient second Ctrl-C
+    # can't orphan those resources.
+    install_interrupt()
+
+    try:
+        if config.server:  # opt-in: drive rollouts through the env-server worker pool
+            from verifiers.v1.cli.eval.runner import run_eval_server
+
+            episodes = asyncio.run(run_eval_server(config))
+        else:  # in-process (default), with or without the live dashboard
+            env = vf.load_environment(config.env)
+            episodes = asyncio.run(run_eval(env, config))
+    except KeyboardInterrupt:
+        # Graceful cleanup has already run (each rollout's `finally`); partial results are on
+        # disk. Exit on the conventional Ctrl-C code without a traceback.
+        raise SystemExit(130)
+    if config.push and not config.rich:
+        from verifiers.v1.utils.platform import push_traces
+
+        push_traces(episodes, config)
+    if not config.rich:  # --rich is the whole output; otherwise dump each trace as JSON
+        for episode in episodes:
+            for trace in episode.traces:
+                print(trace.model_dump_json(indent=2, exclude_none=True))

{verifiers-0.3.1.dev33 → verifiers-0.3.1.dev35}/verifiers/v1/cli/eval/resume.py

@@ -1,9 +1,8 @@
 """Resume an interrupted eval: reload its finished rollouts and run only the rest.
 
-`--resume <dir>` reloads the run's saved config verbatim (so it takes no other
-flags) and writes back into the same dir. `load` keeps the good saved rollouts and
-re-runs what's owed: missing rollouts (never written) and errored ones (dropped and
-redone).
+`--resume` re-enters the run dir the resolved config points at and writes back into
+it. `load` keeps the good saved rollouts and re-runs what's owed: missing rollouts
+(never written) and errored ones (dropped and redone).
 
 A saved rollout is matched to a selected task by content: `task_key` hashes the
 task's wire data. Tasks with identical data are interchangeable, a task whose data
@@ -11,34 +10,18 @@ changed since the interrupted run re-runs, and nothing depends on `data.idx`.
 """
 
 import json
-import tomllib
 from collections import Counter, defaultdict
 from collections.abc import Callable
 from pathlib import Path
 
 from pydantic_core import from_json
 
-from verifiers.v1.cli.output import CONFIG_FILE, TRACES_FILE, sniff_episode
-from verifiers.v1.configs.cli.eval import EvalConfig
+from verifiers.v1.cli.output import TRACES_FILE, sniff_episode
 from verifiers.v1.episode import Episode, WireEpisode
 from verifiers.v1.task import task_key
 from verifiers.v1.trace import WireTrace
 
 
-def load_resume_config(resume_dir: Path) -> EvalConfig:
-    """Rebuild the run's `EvalConfig` from its saved `config.toml`, pointed back at its own
-    output dir so the resumed rollouts append to the same `traces.jsonl`."""
-    config_path = resume_dir / CONFIG_FILE
-    if not config_path.exists():
-        raise SystemExit(
-            f"--resume: no config.toml in {resume_dir} - not an eval output dir"
-        )
-    config = EvalConfig.model_validate(tomllib.loads(config_path.read_text()))
-    config.resume = resume_dir
-    config.output_dir = resume_dir
-    return config
-
-
 def load(
     resume_dir: Path,
     selected_keys: list[str],

{verifiers-0.3.1.dev33 → verifiers-0.3.1.dev35}/verifiers/v1/cli/eval/runner.py

@@ -44,7 +44,7 @@ async def run_eval(env: Env, config: EvalConfig) -> list[Episode]:
     plan = [(task, config.num_rollouts) for task in tasks]
     # Kept on-disk rollouts rejoin the run as finished episodes; only owed ones re-run.
     finished: list[Episode] = []
-    if config.resume is not None:
+    if config.resume:
         keys = [task.hash for task in tasks]
         finished, owed = resume.load(out, keys, config.num_rollouts, env.complete)
         if not owed:  # already complete - report it and exit successfully
@@ -73,7 +73,7 @@ async def run_eval(env: Env, config: EvalConfig) -> list[Episode]:
 
     async def on_complete(episode: Episode) -> None:
         for trace in episode.traces:
-            trace.record_run(EvalRunInfo(id=config.uuid))
+            trace.record_run(EvalRunInfo(id=config.run.id, name=config.run.name))
         await append_episode(out, episode, write_lock)
 
     # Serving resources (shared tool servers, interception) come up once for the
@@ -136,7 +136,7 @@ async def run_eval_server(config: EvalConfig) -> list[Episode]:
     # Spawned processes inherit no logging — hand them the main process's setup so
     # their rollout logs land in the output dir.
     level = "DEBUG" if config.verbose else "INFO"
-    log_file = str(output_path(config) / "eval.log")
+    log_file = str(output_path(config) / "logs" / "eval.log")
     mpctx = mp.get_context("spawn")
     address_queue: mp.Queue = mpctx.Queue()
     # Death pipe: serve_env self-terminates if this process dies abruptly — we keep
@@ -168,7 +168,7 @@ async def run_eval_server(config: EvalConfig) -> list[Episode]:
         ]
         out = output_path(config)
         finished: list[Episode] = []
-        if config.resume is not None:
+        if config.resume:
             keys = [task.hash for task in tasks]
             finished, owed = resume.load(out, keys, config.num_rollouts)
             counts = distribute(keys, owed, config.num_rollouts)
@@ -206,7 +206,7 @@ async def run_eval_server(config: EvalConfig) -> list[Episode]:
                     **payload,
                 )
             for trace in episode.traces:
-                trace.record_run(EvalRunInfo(id=config.uuid))
+                trace.record_run(EvalRunInfo(id=config.run.id, name=config.run.name))
             await append_episode(out, episode, write_lock)
             return [episode]
 

{verifiers-0.3.1.dev33 → verifiers-0.3.1.dev35}/verifiers/v1/cli/gepa.py

@@ -14,7 +14,7 @@ import sys
 from pydantic_config import cli
 
 import verifiers.v1 as vf
-from verifiers.v1.cli.output import output_path, write_config
+from verifiers.v1.cli.output import TRACES_FILE, output_path, write_config
 from verifiers.v1.cli.resolve import (
     extract_id,
     narrow_config,
@@ -69,8 +69,20 @@ def main(argv: list[str] | None = None) -> None:
             "gepa optimizes one agent's prompt against per-trace rewards and can't "
             "drive a multi-agent interaction — only eval runs those"
         )
+    # A named run directory is never silently reused: any write into it — the dry-run
+    # config included, which would destroy the existing traces' config provenance —
+    # would overwrite the previous run.
+    traces_file = output_path(config) / TRACES_FILE
+    if traces_file.exists() and traces_file.stat().st_size > 0:
+        raise SystemExit(
+            f"run directory {output_path(config)} already contains results - "
+            "pick another --run.name or delete it"
+        )
+
     if config.dry_run:  # resolved + validated; write it to the output dir and exit
-        logger.info("wrote config to %s", write_config(config, output_path(config)))
+        logger.info(
+            "wrote config to %s", write_config(config, output_path(config), "gepa.json")
+        )
         return
 
     # First Ctrl-C / SIGTERM warns and raises KeyboardInterrupt so a killed/timed-out run still

{verifiers-0.3.1.dev33 → verifiers-0.3.1.dev35}/verifiers/v1/cli/output.py

@@ -11,63 +11,75 @@ before the episode atom (one bare trace per line) are still readable:
 """
 
 import asyncio
+import hashlib
 import json
 from functools import cache
 from pathlib import Path
 
-import tomli_w
 from pydantic import BaseModel, TypeAdapter
 
 from verifiers.v1.configs.cli.eval import EvalConfig
 from verifiers.v1.episode import Episode, WireEpisode
 from verifiers.v1.trace import Trace
 from verifiers.v1.utils.aio import run_shielded
-from verifiers.v1.utils.install import env_name
 
 TRACES_FILE = "traces.jsonl"
 """Filename a run's rollout episodes are written to (one JSON episode per line)."""
 
-CONFIG_FILE = "config.toml"
-"""Filename a run's resolved config is written to (re-runnable via `@ config.toml`)."""
+CONFIG_DIR = "configs"
+"""Directory inside a run dir holding its resolved config (one `<cli>.json`, re-runnable
+via `@ <run-dir>/configs/<cli>.json`). Resolved configs are JSON, not TOML: JSON keeps
+nulls, so explicit None settings round-trip exactly on re-parse."""
+
+
+def config_digest(config: BaseModel) -> str:
+    """Canonical hash of a resolved config's full model dump (nulls included)."""
+    dump = config.model_dump(mode="json")
+    canonical = json.dumps(dump, sort_keys=True, separators=(",", ":"))
+    return hashlib.sha256(canonical.encode()).hexdigest()
+
+
+def saved_config_path(run_dir: Path) -> Path | None:
+    """The run's saved resolved config (`configs/<cli>.json`), None if absent."""
+    candidates = (
+        sorted((run_dir / CONFIG_DIR).glob("*.json"))
+        if (run_dir / CONFIG_DIR).is_dir()
+        else []
+    )
+    return candidates[0] if candidates else None
+
 
 # Compiling an adapter is the expensive part; run output reuses only a few model classes.
 type_adapter = cache(TypeAdapter)
 
 
 def output_path(config: EvalConfig) -> Path:
-    """Where this run writes: `outputs/<env>--<model>--<harness>/<uuid>` (or the explicit
-    `--output-dir`). The per-run `uuid` leaf means runs never overwrite each other."""
-    if config.output_dir is not None:
-        return config.output_dir
-    taskset = config.env.taskset
-    env = taskset.name if taskset.id else "no-taskset"
-    if taskset.id and config.env.id:
-        # Same compounding as `EnvConfig.env_id`: a `best-of-n+gsm8k` run must
-        # not share a parent dir with a plain `gsm8k` one.
-        env = f"{env_name(config.env.id)}+{env}"
-    # Every seat's resolved harness, distinct, in role order.
-    harness = "+".join(
-        dict.fromkeys(h.name for h in config.env.agent_harnesses().values())
-    )
-    name = f"{env}--{config.model.replace('/', '--')}--{harness or 'default'}"
-    return Path("outputs") / name / config.uuid
-
-
-def write_config(config: BaseModel, results_dir: Path) -> Path:
-    """Write the run's resolved `config.toml` (re-readable via `@ config.toml`); return its
-    path. mode="json" makes values TOML-friendly (Path -> str, etc.); exclude_none drops the
-    nulls TOML can't represent."""
-    results_dir.mkdir(parents=True, exist_ok=True)
-    toml = tomli_w.dumps(config.model_dump(mode="json", exclude_none=True))
-    config_path = results_dir / CONFIG_FILE
-    config_path.write_text(toml)
+    """Where this run writes: `output_dir / run.dir` — the same grouping convention as
+    training. The run directory defaults to the auto-generated run name
+    (`<env>--<model>--<harness>--<short-id>`)."""
+    assert config.run.dir is not None
+    return config.output_dir / config.run.dir
+
+
+def write_config(
+    config: BaseModel, results_dir: Path, filename: str = "eval.json"
+) -> Path:
+    """Write the run's resolved config to `configs/<filename>` (re-readable via
+    `@ <path>`); return its path. The full model dump is written, nulls included, so the
+    file round-trips exactly."""
+    config_dir = results_dir / CONFIG_DIR
+    config_dir.mkdir(parents=True, exist_ok=True)
+    config_path = config_dir / filename
+    config_path.write_text(json.dumps(config.model_dump(mode="json"), indent=2))
     return config_path
 
 
-def save_config(config: BaseModel, results_dir: Path) -> None:
-    """Set up the run's output dir: write `config.toml` and start a fresh (empty)
+def save_config(
+    config: BaseModel, results_dir: Path, filename: str = "eval.json"
+) -> None:
+    """Set up the run's output dir: write the resolved config and start a fresh (empty)
     `traces.jsonl`. Call once up front, before episodes start landing."""
-    write_config(config, results_dir)
+    write_config(config, results_dir, filename)
     (results_dir / TRACES_FILE).write_text(
         ""
     )  # fresh; appended to as rollouts complete

{verifiers-0.3.1.dev33 → verifiers-0.3.1.dev35}/verifiers/v1/cli/replay.py

@@ -10,10 +10,10 @@ cross-trace `score()` can't run offline.
 
 import asyncio
 import contextlib
+import json
 import logging
 import sys
 import time
-import tomllib
 from pathlib import Path
 
 from pydantic_config import cli
@@ -21,10 +21,10 @@ from pydantic_config import cli
 import verifiers.v1 as vf
 from verifiers.v1.cli.dashboard.replay import ReplayProgress, replay_dashboard
 from verifiers.v1.cli.output import (
-    CONFIG_FILE,
     append_trace,
     read_episodes,
     save_config,
+    saved_config_path,
     write_config,
 )
 from verifiers.v1.cli.resolve import narrow_taskset_config
@@ -47,7 +47,7 @@ USAGE = (
 def _narrow(config_path: Path) -> type[ReplayConfig]:
     """Narrow replay config to the saved taskset's config type. The source may be
     an eval run (taskset on the [env] block) or an earlier replay (root taskset)."""
-    data = tomllib.loads(config_path.read_text())
+    data = json.loads(config_path.read_text())
     taskset = data.get("taskset") or (data.get("env") or {}).get("taskset") or {}
     taskset_id = taskset.get("id")
     return narrow_taskset_config(ReplayConfig, taskset_id)
@@ -63,7 +63,10 @@ async def run_replay(config: ReplayConfig, source: Path, out: Path) -> list[Trac
     # Refuse multi-agent up front, from the SOURCE run's saved config — the replay
     # layer's taskset is overridable and must not decide what the run was.
     # Everything below may assume plain single-agent episodes.
-    saved = tomllib.loads((source / CONFIG_FILE).read_text())
+    source_config = saved_config_path(source)
+    if source_config is None:
+        raise SystemExit(f"no saved config under {source} - not a run dir")
+    saved = json.loads(source_config.read_text())
     saved_env = saved.get("env") or {}
     saved_taskset = saved.get("taskset") or saved_env.get("taskset") or {}
     env_cls = vf.environment_class(
@@ -126,7 +129,7 @@ async def run_replay(config: ReplayConfig, source: Path, out: Path) -> list[Trac
         for _ in range(config.num_rescores)
     ]
 
-    save_config(config, out)
+    save_config(config, out, "replay.json")
     logger.info(
         "replay: re-scoring %d trace(s) x%d from %s -> %s",
         len(traces),
@@ -201,15 +204,15 @@ def main(argv: list[str] | None = None) -> None:
         cli(ReplayConfig)  # full, typed pydantic-config option help
         return
     source = Path(argv.pop(0))
-    config_path = source / CONFIG_FILE
-    if not config_path.exists():
-        raise SystemExit(f"{USAGE}\nno config.toml in {source}")
+    config_path = saved_config_path(source)
+    if config_path is None:
+        raise SystemExit(f"{USAGE}\nno saved config under {source} - not a run dir")
 
     layered = ["@", str(config_path), *argv]
     config_type = _narrow(config_path)
     sys.argv = [sys.argv[0], *layered]
     config = cli(config_type)
-    source_out = tomllib.loads(config_path.read_text()).get("output_dir")
+    source_out = json.loads(config_path.read_text()).get("output_dir")
     # Clear the source run's output_dir unless the user overrode it.
     if config.output_dir is None or str(config.output_dir) == str(source_out):
         config = config.model_copy(update={"output_dir": None})
@@ -222,10 +225,10 @@ def main(argv: list[str] | None = None) -> None:
     level = "DEBUG" if config.verbose else "INFO"
     if config.dry_run:
         setup_logging(level)
-        logger.info("wrote config to %s", write_config(config, out))
+        logger.info("wrote config to %s", write_config(config, out, "replay.json"))
         return
 
-    log_file = str(out / "replay.log")
+    log_file = str(out / "logs" / "replay.log")
     if config.rich:
         setup_logging(level, log_file=log_file, console=False)
         logging.lastResort = None

verifiers-0.3.1.dev35/verifiers/v1/cli/resume.py

@@ -0,0 +1,15 @@
+"""Resume primitives shared by eval-like CLIs."""
+
+
+def distribute(
+    selected_keys: list[str], owed: dict[str, int], num_results: int
+) -> list[int]:
+    """Spread each key's owed results over its selected instances, in order."""
+    remaining = dict(owed)
+    counts: list[int] = []
+    for key in selected_keys:
+        take = min(num_results, remaining.get(key, 0))
+        if take:
+            remaining[key] -= take
+        counts.append(take)
+    return counts