verifiers 0.3.2.dev99__tar.gz → 0.3.2.dev101__tar.gz

{verifiers-0.3.2.dev99 → verifiers-0.3.2.dev101}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.3.2.dev99
+Version: 0.3.2.dev101
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.3.2.dev99 → verifiers-0.3.2.dev101}/pyproject.toml

@@ -152,12 +152,12 @@ prime-pydantic-config = false
 renderers = false
 
 [project.scripts]
-eval = "verifiers.v1.cli.eval.main:main"
-validate = "verifiers.v1.cli.validate:main"
-debug = "verifiers.v1.cli.debug:main"
-replay = "verifiers.v1.cli.replay:main"
-init = "verifiers.v1.cli.init:main"
-gepa = "verifiers.v1.cli.gepa:main"
+vf-eval = "verifiers.v1.cli.eval.main:main"
+vf-validate = "verifiers.v1.cli.validate:main"
+vf-debug = "verifiers.v1.cli.debug:main"
+vf-replay = "verifiers.v1.cli.replay:main"
+vf-init = "verifiers.v1.cli.init:main"
+vf-gepa = "verifiers.v1.cli.gepa:main"
 
 # hatchling configuration
 [tool.hatch.version]

{verifiers-0.3.2.dev99 → verifiers-0.3.2.dev101}/tests/v1/conftest.py

@@ -5,7 +5,8 @@ settings that still exercise the path, then assert on the resulting `Trace`(s)
 not unit tests of individual components. They need a model API key (`PRIME_API_KEY`);
 without one the `e2e`-marked tests skip (config parsing still runs).
 
-`run_v1` mirrors the eval CLI's in-process path (`run_eval` with `--no-serve`). Placement coverage (harness x harness runtime x tool
+`run_v1` mirrors the eval CLI (`run_eval`, in-process); `run_v1_server` drives the same env
+through an env-server worker pool, the path prime-rl trains through. Placement coverage (harness x harness runtime x tool
 server runtime) is PAIRWISE, not a full cross product: each test carries a curated list of
 combinations (in test_e2e.py) that hits every axis value and the cross-boundary pairs with
 distinct networking. The full cross bought flake exposure and CI minutes, not coverage — add
@@ -32,6 +33,7 @@ prime/modal provision real remote sandboxes (slow, infra-flaky, need setup), so
 CI runs deterministic tests across the Python matrix and the remaining live E2Es once.
 """
 
+import asyncio
 import os
 from pathlib import Path
 
@@ -130,9 +132,7 @@ def _eval_config(
     taskset_overrides: dict | None = None,
     runtime: dict | None = None,
     env: dict | None = None,
-    pool: dict | None = None,
     reasoning_effort: str | None = None,
-    server: bool = False,
 ) -> EvalConfig:
     """Build the smallest `EvalConfig` that still exercises the path, shared by the in-process
     (`run_v1`) and env-server (`run_v1_server`) fixtures. `taskset_overrides` merges onto the
@@ -184,7 +184,6 @@ def _eval_config(
             "reasoning_effort": reasoning_effort,
         },
         rich=None,
-        serve=({"pool": pool} if pool else {}) if server else None,
         output_dir=output_dir.parent,
         run={"dir": output_dir.name},
         model=CI_MODEL,
@@ -194,8 +193,7 @@ def _eval_config(
 
 @pytest.fixture
 def run_v1():
-    """Run a v1 taskset end-to-end in-process (`run_eval` with `--no-serve`) and return
-    its traces."""
+    """Run a v1 taskset end-to-end in-process (`run_eval`) and return its traces."""
 
     async def _run(taskset: str, **kwargs) -> list[Trace]:
         config = _eval_config(taskset, **kwargs)
@@ -208,17 +206,56 @@ def run_v1():
 
 @pytest.fixture
 def run_v1_server():
-    """Run a v1 taskset through the env-server worker pool (`run_eval`'s default path) —
-    the path a CLI run and prime-rl training both take. Spawns the broker + a worker, so
-    it's the only fixture that exercises serving resources (shared tool servers,
-    interception pool) being stood up by the *server* rather than the in-process runner.
-    Pinned to a single static worker for determinism."""
+    """Run a v1 taskset through an env-server worker pool — the path prime-rl trains
+    through. Spawns the broker + a worker, so it's the only fixture that exercises
+    serving resources (shared tool servers, interception pool) being stood up by the
+    *server* rather than the in-process runner. Pinned to a single static worker for
+    determinism."""
+    import multiprocessing as mp
+
+    from verifiers.v1.serve import EnvClient, env_config_data, serve_env
+    from verifiers.v1.utils.loaders import load_taskset
 
     async def _run(taskset: str, **kwargs) -> list[Trace]:
-        kwargs.setdefault("pool", {"type": "static", "num_workers": 1})
-        config = _eval_config(taskset, server=True, **kwargs)
-        records = await run_eval(config)
-        return [t for r in records for t in r.traces]
+        config = _eval_config(taskset, **kwargs)
+        tasks = list(load_taskset(config.env.taskset).head(config.num_tasks))
+        mpctx = mp.get_context("spawn")
+        address_queue: mp.Queue = mpctx.Queue()
+        proc = mpctx.Process(
+            target=serve_env,
+            kwargs={
+                "max_workers": 1,
+                "elastic": False,
+                "address": "tcp://127.0.0.1:0",
+                "address_queue": address_queue,
+                "config_data": env_config_data(config.env),
+                "max_concurrent": config.max_concurrent,
+            },
+        )
+        proc.start()
+        try:
+            address = await asyncio.to_thread(address_queue.get, timeout=600)
+            client = EnvClient(address=address)
+            try:
+                await client.wait_for_server_startup(timeout=600)
+                episodes = await asyncio.gather(
+                    *(
+                        client.run(
+                            client=config.client,
+                            model=config.model,
+                            sampling=config.sampling,
+                            task_data=task.data.model_dump(mode="json"),
+                        )
+                        for task in tasks
+                        for _ in range(config.num_rollouts)
+                    )
+                )
+            finally:
+                await client.close()
+        finally:
+            proc.terminate()
+            await asyncio.to_thread(proc.join, 10)
+        return [t for e in episodes for t in e.traces]
 
     return _run
 

{verifiers-0.3.2.dev99 → verifiers-0.3.2.dev101}/tests/v1/test_configs.py

@@ -1,7 +1,7 @@
 """Every checked-in v1 eval config parses.
 
 Mirrors prime-rl's config test: glob the configs and assert each validates into its config
-type. The root `configs/*.toml` are the `uv run eval @ <file>` v1 configs (EvalConfig).
+type. The root `configs/*.toml` are the `uv run vf-eval @ <file>` v1 configs (EvalConfig).
 """
 
 import tomllib

{verifiers-0.3.2.dev99 → verifiers-0.3.2.dev101}/tests/v1/test_envs.py

@@ -66,7 +66,7 @@ def test_eval(taskset: str):
         for flag in (f"--env.{seat}.max-turns", "4")
     ]
     cmd = [
-        "uv", "run", "--no-sync", "eval", taskset,
+        "uv", "run", "--no-sync", "vf-eval", taskset,
         *model,
         "-n", "1", "-r", "1", *caps,
         "--sampling.max-tokens", "512", "--no-rich", "--no-push",

verifiers-0.3.2.dev101/tests/v1/test_serve_delta.py

@@ -0,0 +1,266 @@
+"""The env-serve delta stream: a trace's changes leave once each and reassemble into
+the episode the worker finished with."""
+
+import asyncio
+import copy
+
+import pytest
+
+import verifiers.v1 as vf
+from verifiers.v1.episode import EnvInfo, Episode, WireEpisode
+from verifiers.v1.graph import MessageNode
+from verifiers.v1.semantic import ParentLink
+from verifiers.v1.serve.delta import (
+    DeltaStreamer,
+    EpisodeAssembly,
+    TraceSummary,
+    dump,
+    pack,
+    unpack,
+)
+from verifiers.v1.trace import ModelCall, TimeSpan, TraceTask
+from verifiers.v1.types import AssistantMessage, Usage, UserMessage
+
+
+class MyTask(vf.TaskData):
+    prompt: str
+    answer: str
+
+
+class Slot:
+    def __init__(self) -> None:
+        self.traces: list[vf.Trace] = []
+
+
+async def settle() -> None:
+    """Let a notified change reach the wire: the flush is a callback that starts a task."""
+    for _ in range(3):
+        await asyncio.sleep(0)
+
+
+def make_trace() -> vf.Trace:
+    return vf.Trace(
+        agent=vf.AgentInfo(config=vf.AgentConfig()),
+        task=TraceTask(type="MyTask", data=MyTask(idx=0, prompt="q", answer="a")),
+    )
+
+
+def add_turn(trace: vf.Trace, reply: str) -> None:
+    parent = len(trace.nodes) - 1 if trace.nodes else None
+    if parent is None:
+        trace.nodes.append(MessageNode(parent=None, message=UserMessage(content="q")))
+        parent = 0
+    trace.nodes.append(
+        MessageNode(
+            parent=parent,
+            message=AssistantMessage(content=reply),
+            sampled=True,
+            token_ids=[1, 2, 3],
+            mask=[False, True, True],
+            logprobs=[-0.1, -0.2],
+        )
+    )
+    trace.calls.append(
+        ModelCall(
+            node=len(trace.nodes) - 1,
+            usage=Usage(prompt_tokens=3, completion_tokens=2),
+            time=TimeSpan(start=1.0, end=2.0),
+        )
+    )
+    trace.notify()
+
+
+def test_delta_fields_cover_every_serialized_trace_field():
+    """A new Trace field must be routed through the stream, or it silently vanishes."""
+    from verifiers.v1.serve.delta import HEADER_FIELDS, LIST_FIELDS, SCALAR_FIELDS
+
+    streamed = set(HEADER_FIELDS) | set(LIST_FIELDS) | set(SCALAR_FIELDS)
+    serialized = {
+        name for name, info in vf.Trace.model_fields.items() if not info.exclude
+    }
+    assert streamed == serialized
+
+
+@pytest.mark.asyncio
+async def test_failed_send_is_diffed_again():
+    """A delta the wire refused is not lost: its cursor stays and the next flush
+    carries the same content, so the client still assembles the whole trace."""
+    slot = Slot()
+    frames: list[bytes] = []
+    fail = {"on": False}
+
+    async def send(data: bytes) -> None:
+        if fail["on"]:
+            raise OSError("host unreachable")
+        frames.append(data)
+
+    async with DeltaStreamer(slot, send) as streamer:
+        trace = make_trace()
+        slot.traces.append(trace)
+        streamer.watch(trace)
+        await settle()
+        fail["on"] = True
+        add_turn(trace, "a1")
+        await settle()
+        fail["on"] = False
+        add_turn(trace, "a2")
+        await settle()
+        slot.traces = [trace]
+    assembly = EpisodeAssembly()
+    for frame in frames:
+        assembly.apply(unpack(frame))
+    assert len(assembly.traces[trace.id]["nodes"]) == 3
+    assert len(assembly.traces[trace.id]["calls"]) == 2
+
+
+@pytest.mark.asyncio
+async def test_pending_preview_streams_and_clears_on_commit():
+    slot = Slot()
+    frames: list[bytes] = []
+
+    async def send(data: bytes) -> None:
+        frames.append(data)
+
+    async with DeltaStreamer(slot, send) as streamer:
+        trace = make_trace()
+        slot.traces.append(trace)
+        streamer.watch(trace)
+        add_turn(trace, "a1")
+        await settle()
+        # the harness sends its next request: the tool result is previewed at once
+        request, subagent = ["request"], ["sub-agent"]  # unhashable, like a PendingTurn
+        trace.preview(request, [UserMessage(content="tool says 42")])
+        await settle()
+        preview = unpack(frames[-1])
+        assert preview["pending"][0]["content"] == "tool says 42"
+        assert "nodes" not in preview
+        # a concurrent request (a sub-agent) previews under its own key; its failure
+        # takes only its own messages back
+        trace.preview(subagent, [UserMessage(content="sub-agent asks")])
+        await settle()
+        assert [m["content"] for m in unpack(frames[-1])["pending"]] == [
+            "tool says 42",
+            "sub-agent asks",
+        ]
+        trace.clear_preview(subagent)
+        trace.notify()
+        await settle()
+        assert [m["content"] for m in unpack(frames[-1])["pending"]] == ["tool says 42"]
+        # the model answers: the turn commits and the preview goes with it
+        trace.nodes.append(
+            MessageNode(parent=1, message=UserMessage(content="tool says 42"))
+        )
+        trace.clear_preview(request)
+        add_turn(trace, "a2")
+        await settle()
+        committed = unpack(frames[-1])
+        assert len(committed["nodes"]) == 2 and "pending" not in committed
+        trace.stop("agent_completed")
+        trace.ok = True
+        episode = Episode(
+            env=EnvInfo(id="my-env"), task=trace.task, ok=True, traces=[trace]
+        )
+        slot.traces = list(episode.traces)
+    assembly = EpisodeAssembly()
+    for frame in frames:
+        assembly.apply(unpack(frame))
+        if unpack(frame).get("pending"):
+            assert assembly.traces[trace.id]["pending"][0]["content"] == "tool says 42"
+    assert assembly.traces[trace.id]["pending"] == []
+    summaries = [
+        TraceSummary(id=trace.id, nodes=len(trace.nodes), calls=len(trace.calls))
+    ]
+    record = assembly.finish(dump(episode, exclude={"traces"}), summaries)
+    assert "pending" not in record["traces"][0]
+    assert WireEpisode.model_validate(unpack(pack(record))).traces[0].id == trace.id
+
+
+@pytest.mark.asyncio
+async def test_deltas_stream_once_and_reassemble_the_episode():
+    slot = Slot()
+    frames: list[bytes] = []
+
+    async def send(data: bytes) -> None:
+        frames.append(data)
+
+    async with DeltaStreamer(slot, send) as streamer:
+        trace = make_trace()
+        slot.traces.append(trace)
+        streamer.watch(trace)
+        trace.timing.boot.start = 1.0
+        trace.notify()
+        await settle()
+        assert len(frames) == 1, "the mint and the boot span coalesce into one delta"
+        add_turn(trace, "a1")
+        await settle()
+        add_turn(trace, "a2")
+        trace.nodes[0].semantic_parents.append(ParentLink(node=1, type="reply"))
+        await settle()
+        trace.record_reward("match", 1.0)
+        trace.stop("agent_completed")
+        trace.ok = True
+        episode = Episode(
+            env=EnvInfo(id="my-env"), task=trace.task, ok=True, traces=[trace]
+        )
+        slot.traces = list(episode.traces)
+    # The exit flushed the tail (reward, stop, ok) without re-sending any node.
+    deltas = [unpack(frame) for frame in frames]
+    sent_nodes = sum(len(delta.get("nodes", [])) for delta in deltas)
+    sent_calls = sum(len(delta.get("calls", [])) for delta in deltas)
+    assert (sent_nodes, sent_calls) == (3, 2)
+    assert sum("open" in delta for delta in deltas) == 1
+    assert any("links" in delta for delta in deltas)
+    assert deltas[-1]["set"]["stop_condition"] == "agent_completed"
+
+    handed = copy.deepcopy(deltas)
+    assembly = EpisodeAssembly()
+    for delta in deltas:
+        assembly.apply(delta)
+    assert deltas == handed, "assembling never mutates a delta the caller holds"
+    head = dump(episode, exclude={"traces"})
+    summaries = [
+        TraceSummary(id=trace.id, nodes=len(trace.nodes), calls=len(trace.calls))
+    ]
+    rebuilt = WireEpisode.model_validate(unpack(pack(assembly.finish(head, summaries))))
+    assert rebuilt.ok and rebuilt.traces[0].id == trace.id
+    assert rebuilt.traces[0].messages == trace.messages
+    assert (
+        rebuilt.traces[0].nodes[0].semantic_parents == trace.nodes[0].semantic_parents
+    )
+    assert rebuilt.traces[0].rewards["match"].value == 1.0
+    assert rebuilt.traces[0].stop_condition == "agent_completed"
+    assert len(rebuilt.traces[0].calls) == 2
+
+
+@pytest.mark.asyncio
+async def test_discarded_attempt_drops_its_trace():
+    slot = Slot()
+    frames: list[bytes] = []
+
+    async def send(data: bytes) -> None:
+        frames.append(data)
+
+    async with DeltaStreamer(slot, send) as streamer:
+        first = make_trace()
+        slot.traces.append(first)
+        streamer.watch(first)
+        add_turn(first, "a1")
+        await settle()
+        slot.traces = []  # the attempt is retried
+        second = make_trace()
+        slot.traces.append(second)
+        streamer.watch(second)
+        await settle()
+    deltas = [unpack(frame) for frame in frames]
+    assert {"trace": first.id, "discard": True} in deltas
+    assembly = EpisodeAssembly()
+    for delta in deltas:
+        assembly.apply(delta)
+    assert list(assembly.traces) == [second.id]
+
+
+def test_finish_refuses_a_gap():
+    assembly = EpisodeAssembly()
+    assembly.apply({"trace": "t", "open": {"id": "t"}, "nodes": [{}]})
+    with pytest.raises(RuntimeError, match="assembled 1 nodes"):
+        assembly.finish({}, [TraceSummary(id="t", nodes=2, calls=0)])

{verifiers-0.3.2.dev99 → verifiers-0.3.2.dev101}/verifiers/v1/agent.py

@@ -54,11 +54,19 @@ __all__ = ["Agent", "AgentConfig", "Agents", "TimeoutConfig", "make_agent"]
 logger = logging.getLogger(__name__)
 
 
+DEFAULT_ROLLOUT_TIMEOUT = 4 * 3600.0
+"""Agent solve-attempt budget when neither the eval config nor the task sets one."""
+
+
 def resolve_rollout_timeouts(timeout: TimeoutConfig, task: Task) -> RolloutTimeouts:
     """Apply an agent's stage-timeout precedence to one task."""
     agent_timeout = (
         timeout.rollout if timeout.rollout is not None else task.data.timeout.agent
     )
+    if agent_timeout is None:
+        agent_timeout = DEFAULT_ROLLOUT_TIMEOUT
+    elif agent_timeout == 0:
+        agent_timeout = None  # explicit: unbounded
     return RolloutTimeouts(
         setup=timeout.setup if timeout.setup is not None else task.data.timeout.setup,
         agent=agent_timeout,

{verifiers-0.3.2.dev99 → verifiers-0.3.2.dev101}/verifiers/v1/cli/dashboard/eval.py

@@ -16,6 +16,7 @@ from rich.table import Table
 from rich.text import Text
 
 from verifiers.v1.cli.dashboard.base import live_view
+from verifiers.v1.cli.eval.hint import PRIME_RL_HINT
 from verifiers.v1.cli.output import attempt_log_file, output_path
 from verifiers.v1.configs.cli.eval import EvalConfig
 from verifiers.v1.env import RunSlot
@@ -280,6 +281,10 @@ def Overview(config: EvalConfig) -> Table:
     return grid
 
 
+def _prime_rl_footer() -> Group:
+    return Group(Text(""), Text(PRIME_RL_HINT, style="dim", overflow="fold"))
+
+
 def _push_footer(push: "PushState | None") -> Group | None:
     """The `--push` line under the rollouts: dim with the run's URL while it streams,
     white once pushed, red when it failed. `None` when `--push` is off or the run stayed
@@ -833,13 +838,15 @@ def _render(
     # The --push status line (and, on Ctrl-C, the cleanup notice) appear under the rollouts. Measure
     # the fixed top (header + progress + rule) and the footer so the rollout rows fill what's left;
     # page through them (timer / arrows) when they'd overflow (else rich truncates).
-    footers = [f for f in (_push_footer(push), _interrupt_footer()) if f is not None]
-    footer = Group(*footers) if footers else None
+    footers = [
+        f
+        for f in (_push_footer(push), _interrupt_footer(), _prime_rl_footer())
+        if f is not None
+    ]
+    footer = Group(*footers)
     progress = Progress(slots, start, completed)
     top = Group(header, progress, Rule(style="dim"))
-    reserved = len(_CONSOLE.render_lines(top))
-    if footer is not None:
-        reserved += len(_CONSOLE.render_lines(footer))
+    reserved = len(_CONSOLE.render_lines(top)) + len(_CONSOLE.render_lines(footer))
     rows_per_page = max(1, _CONSOLE.size.height - reserved - 1)
     if tail is not None:  # --show-logs: the run's log stream in place of rollout rows
         parts = [
@@ -847,9 +854,8 @@ def _render(
             progress,
             Rule(style="dim"),
             tail.view(rows_per_page),
+            footer,
         ]
-        if footer is not None:
-            parts.append(footer)
         return Group(*parts)
     page_groups, index, count = _paginate(_groups(slots), rows_per_page, pager, now)
     if count > 1:
@@ -859,9 +865,8 @@ def _render(
         progress,
         Rule(style="dim"),
         Rows(page_groups, now, runtime_type, completed),
+        footer,
     ]
-    if footer is not None:
-        parts.append(footer)
     return Group(*parts)
 
 

{verifiers-0.3.2.dev99 → verifiers-0.3.2.dev101}/verifiers/v1/cli/debug.py

@@ -36,7 +36,7 @@ from verifiers.v1.utils.logging import setup_logging
 logger = logging.getLogger(__name__)
 
 USAGE = (
-    "usage: uv run debug [<taskset-id>] (--command <cmd> | --script-path <path>) "
+    "usage: uv run vf-debug [<taskset-id>] (--command <cmd> | --script-path <path>) "
     "[--runtime.type subprocess] [options] [@ file.toml]\n"
     "       runs setup, then one command or uploaded host script, and saves traces"
 )

verifiers-0.3.2.dev101/verifiers/v1/cli/eval/hint.py

@@ -0,0 +1,5 @@
+PRIME_RL_HINT = (
+    "vf-eval will be deprecated - move to prime-rl's `uv run eval`: it streams traces "
+    "live into a dashboard, evaluates several envs in one run and scales through env "
+    "servers. https://github.com/PrimeIntellect-ai/prime-rl"
+)

{verifiers-0.3.2.dev99 → verifiers-0.3.2.dev101}/verifiers/v1/cli/eval/main.py

@@ -30,8 +30,8 @@ from verifiers.v1.utils.logging import setup_logging
 logger = logging.getLogger(__name__)
 
 USAGE = (
-    "usage: uv run eval [<taskset-id>] [--env.id <id>] [options] [@ file.toml]\n"
-    "       uv run eval @ <run-dir>/configs/resolved/eval.json --resume   (re-run the run's missing/errored rollouts)"
+    "usage: uv run vf-eval [<taskset-id>] [--env.id <id>] [options] [@ file.toml]\n"
+    "       uv run vf-eval @ <run-dir>/configs/resolved/eval.json --resume   (re-run the run's missing/errored rollouts)"
 )
 
 
@@ -48,7 +48,7 @@ def main(argv: list[str] | None = None) -> None:
         return
     # An env-block flag skips the usage gate so the typed parse renders its
     # did-you-mean instead of a bare usage line.
-    typed_axis = any(a.startswith(("--env.", "--serve.")) for a in argv)
+    typed_axis = any(a.startswith("--env.") for a in argv)
     if (
         not extract_id(argv, "env.taskset")
         and not references_config_file(argv)
@@ -108,7 +108,7 @@ def main(argv: list[str] | None = None) -> None:
             raise SystemExit(
                 f"--resume requires the exact config the run was started with - it "
                 f"differs in [{', '.join(changed)}]. Resumed rollouts would not be "
-                f"comparable; re-run with `uv run eval @ {saved_path} --resume`, or "
+                f"comparable; re-run with `uv run vf-eval @ {saved_path} --resume`, or "
                 "start a fresh run"
             )
     if config.dry_run:  # resolved + validated; write it to the output dir and exit
@@ -116,8 +116,7 @@ def main(argv: list[str] | None = None) -> None:
         logger.info("wrote config to %s", write_config(config, run_path))
         return
     # Always tee this attempt's logs to `logs/attempt_<n>/eval.log` (`logs/latest`
-    # points there) — in server mode (the default) the workers write there too, and
-    # `--rich.show-logs` tails it live.
+    # points there); `--rich.show-logs` tails it live.
     log_file = str(create_attempt_log_dir(run_path) / "eval.log")
     level = "DEBUG" if config.verbose else "INFO"
     setup_logging(level, log_file=log_file, console=config.rich is None)
@@ -128,7 +127,6 @@ def main(argv: list[str] | None = None) -> None:
     install_interrupt()
 
     try:
-        # Through the env-server worker pool by default; in-process with --no-serve.
         episodes = asyncio.run(run_eval(config))
     except KeyboardInterrupt:
         # Graceful cleanup has already run (each rollout's `finally`); partial results are on