verifiers 0.3.2.dev72__tar.gz → 0.3.2.dev74__tar.gz

{verifiers-0.3.2.dev72 → verifiers-0.3.2.dev74}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.3.2.dev72
+Version: 0.3.2.dev74
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers
@@ -35,6 +35,7 @@ Requires-Dist: msgpack>=1.1.2
 Requires-Dist: numpy>=2.1.0
 Requires-Dist: openai<3.0.0,>=2.54.0
 Requires-Dist: prime-pydantic-config[toml]>=0.4.3
+Requires-Dist: prime-runs>=0.1.2
 Requires-Dist: prime-sandboxes>=0.2.39
 Requires-Dist: prime-tunnel>=0.1.8
 Requires-Dist: pydantic>=2.12.3

{verifiers-0.3.2.dev72 → verifiers-0.3.2.dev74}/pyproject.toml

@@ -34,6 +34,7 @@ dependencies = [
     "openai>=2.54.0,<3.0.0",
     "prime-tunnel>=0.1.8",
     "prime-sandboxes>=0.2.39",
+    "prime-runs>=0.1.2",
     "pydantic>=2.12.3",
     "rich>=11.0.0",
     "tenacity>=8.5.0",
@@ -146,6 +147,7 @@ ty = "2026-07-28T00:00:00Z"
 # PrimeIntellect-published on PyPI (trusted publisher)
 prime-tunnel = false
 prime-sandboxes = false
+prime-runs = false
 prime-pydantic-config = false
 renderers = false
 

{verifiers-0.3.2.dev72 → verifiers-0.3.2.dev74}/tests/v1/conftest.py

@@ -188,6 +188,7 @@ def _eval_config(
         output_dir=output_dir.parent,
         run={"dir": output_dir.name},
         model=CI_MODEL,
+        push=False,
     )
 
 

verifiers-0.3.2.dev74/tests/v1/test_platform_run.py

@@ -0,0 +1,104 @@
+"""`open_run`: the eval's run on the platform, and `run.attach` for hosted evaluations."""
+
+import prime_runs as pr
+import pytest
+
+from verifiers.v1.configs.cli.eval import EvalConfig
+from verifiers.v1.utils import platform
+from verifiers.v1.utils.platform import PushState, open_run
+
+
+class _FakeRun:
+    """Stands in for `pr.Run`: an id, a URL, nothing else the tests reach for."""
+
+    def __init__(self, run_id: str, mode: str) -> None:
+        self.id = run_id
+        self.mode = mode
+        self.url = (
+            None if mode == "disabled" else f"https://app.example/evaluations/{run_id}"
+        )
+        self.finished = False
+
+
+@pytest.fixture
+def init_calls(monkeypatch):
+    """Record every `pr.init` call; the fake returns a run keyed by the attach id."""
+    calls: list[dict] = []
+
+    def fake_init(**kwargs):
+        calls.append(kwargs)
+        return _FakeRun(kwargs.get("id") or "eval-created", kwargs["mode"])
+
+    monkeypatch.setattr(platform.pr, "init", fake_init)
+    monkeypatch.delenv(pr.MODE_ENV, raising=False)
+    return calls
+
+
+def test_attach_needs_push():
+    with pytest.raises(ValueError, match="needs push"):
+        EvalConfig(run={"attach": "eval-hosted"}, push=False)
+
+
+def test_a_run_without_attach_is_created_on_the_platform(init_calls):
+    config = EvalConfig(push=True)
+    state = PushState()
+
+    run = open_run(config, state, num_examples=3)
+
+    assert init_calls[0]["mode"] == "online"
+    assert init_calls[0]["id"] is None
+    assert run.id == config.run.id == "eval-created"
+
+
+def test_attach_hands_the_launchers_id_to_the_sdk(init_calls):
+    config = EvalConfig(run={"attach": "eval-hosted"})
+    state = PushState()
+
+    run = open_run(config, state, num_examples=3)
+
+    assert init_calls[0]["mode"] == "online"
+    assert init_calls[0]["id"] == "eval-hosted"
+    # The launcher's id is the run's one id: traces and the run dir carry it too.
+    assert run.id == config.run.id == "eval-hosted"
+    assert state.error is None
+
+
+def test_a_failed_attach_fails_the_eval_instead_of_falling_back(monkeypatch):
+    def refuse(**kwargs):
+        raise pr.UnauthorizedError("bad key")
+
+    monkeypatch.setattr(platform.pr, "init", refuse)
+    monkeypatch.delenv(pr.MODE_ENV, raising=False)
+    config = EvalConfig(run={"attach": "eval-hosted"})
+
+    with pytest.raises(RuntimeError, match="could not attach to run 'eval-hosted'"):
+        open_run(config, PushState(), num_examples=3)
+
+
+def test_a_failed_open_without_attach_still_falls_back_to_a_local_run(monkeypatch):
+    calls: list[dict] = []
+
+    def flaky(**kwargs):
+        calls.append(kwargs)
+        if kwargs["mode"] == "online":
+            raise pr.UnauthorizedError("bad key")
+        return _FakeRun("local-run", kwargs["mode"])
+
+    monkeypatch.setattr(platform.pr, "init", flaky)
+    monkeypatch.delenv(pr.MODE_ENV, raising=False)
+    state = PushState()
+
+    run = open_run(EvalConfig(push=True), state, num_examples=3)
+
+    assert [call["mode"] for call in calls] == ["online", "disabled"]
+    assert run.mode == "disabled"
+    assert state.error is not None and "UnauthorizedError" in state.error
+
+
+def test_the_sdk_kill_switch_refuses_an_attach(init_calls, monkeypatch):
+    monkeypatch.setenv(pr.MODE_ENV, "disabled")
+    config = EvalConfig(run={"attach": "eval-hosted"})
+
+    with pytest.raises(RuntimeError, match="PRIME_RUNS_MODE=disabled"):
+        open_run(config, PushState(), num_examples=3)
+    assert init_calls == []

{verifiers-0.3.2.dev72 → verifiers-0.3.2.dev74}/verifiers/v1/cli/dashboard/eval.py

@@ -281,17 +281,25 @@ def Overview(config: EvalConfig) -> Table:
 
 
 def _push_footer(push: "PushState | None") -> Group | None:
-    """The `--push` status line under the rollouts, shown once the run finishes and the upload
-    begins: dim `Pushing traces...` while it runs, then white `Traces pushed (<url>)` or red
-    `Trace push failed (<err>)`. `None` (no line) until the upload starts and when `--push` is off."""
+    """The `--push` line under the rollouts: dim with the run's URL while it streams,
+    white once pushed, red when it failed. `None` when `--push` is off or the run stayed
+    local."""
     if push is None or not push.started:
         return None
-    if not push.done:
-        line = Text("Pushing traces...", style="dim")
-    elif push.url:
+    if push.error and push.url:
+        # The run exists and holds what streamed up; only closing it out failed.
         line = Text(f"Traces pushed ({push.url})", style="white", overflow="fold")
-    else:
+        line.append(f"  not closed out: {push.error}", style="red")
+    elif push.error:
         line = Text(f"Trace push failed ({push.error})", style="red", overflow="fold")
+    elif push.incomplete and push.url:
+        # Closed out, but the uploader lost records: what landed is there, say what didn't.
+        line = Text(f"Traces pushed ({push.url})", style="white", overflow="fold")
+        line.append(f"  incomplete: {push.incomplete}", style="red")
+    elif not push.finished:
+        line = Text(f"Pushing traces ({push.url})", style="dim", overflow="fold")
+    else:
+        line = Text(f"Traces pushed ({push.url})", style="white", overflow="fold")
     return Group(Rule(style="dim"), line)
 
 

{verifiers-0.3.2.dev72 → verifiers-0.3.2.dev74}/verifiers/v1/cli/eval/main.py

@@ -134,10 +134,6 @@ def main(argv: list[str] | None = None) -> None:
         # Graceful cleanup has already run (each rollout's `finally`); partial results are on
         # disk. Exit on the conventional Ctrl-C code without a traceback.
         raise SystemExit(130)
-    if config.push and config.rich is None:
-        from verifiers.v1.utils.platform import push_traces
-
-        push_traces(episodes, config)
     if (
         config.rich is None
     ):  # --rich is the whole output; otherwise dump each trace as JSON

{verifiers-0.3.2.dev72 → verifiers-0.3.2.dev74}/verifiers/v1/cli/eval/runner.py

@@ -13,8 +13,8 @@ import asyncio
 import contextlib
 import logging
 import time
-from collections.abc import AsyncIterator, Awaitable, Callable
-from typing import cast
+from collections.abc import AsyncIterator, Awaitable, Callable, Iterable
+from typing import TypeVar, cast
 
 from verifiers.v1.cli.dashboard import dashboard
 from verifiers.v1.cli.eval import resume
@@ -30,9 +30,36 @@ from verifiers.v1.configs.cli.eval import EvalConfig
 from verifiers.v1.configs.serve import ServeConfig
 from verifiers.v1.env import Env, RunSlot
 from verifiers.v1.episode import Episode, EvalRunInfo
+from verifiers.v1.utils.aio import run_shielded
+from verifiers.v1.utils.platform import (
+    PushState,
+    abort_run,
+    finish_run,
+    log_episodes,
+    open_run,
+)
 
 logger = logging.getLogger(__name__)
 
+T = TypeVar("T")
+
+
+async def gather_rollouts(rollouts: Iterable[Awaitable[T]]) -> list[T]:
+    """`asyncio.gather`, but one rollout failing cancels the rest and waits for them
+    to unwind, so nothing keeps uploading into a run the caller is already closing.
+    Not a `TaskGroup`: that wraps errors in an `ExceptionGroup`, and `main` would no
+    longer see a `KeyboardInterrupt` as Ctrl-C."""
+    tasks = [asyncio.ensure_future(rollout) for rollout in rollouts]
+    try:
+        return await asyncio.gather(*tasks)
+    except BaseException:
+        for task in tasks:
+            task.cancel()
+        # return_exceptions: wait for every task, not just the first to cancel.
+        await asyncio.gather(*tasks, return_exceptions=True)
+        raise
+
+
 RunSlotFn = Callable[[RunSlot], Awaitable[Episode]]
 OnComplete = Callable[[Episode], Awaitable[None]]
 
@@ -203,47 +230,55 @@ async def run_eval(config: EvalConfig) -> list[Episode]:
         asyncio.Semaphore(config.max_concurrent) if config.max_concurrent else None
     )
     write_lock = asyncio.Lock()
+    push_state = PushState()
+
+    # Opened before the first rollout so every episode streams as it lands.
+    run = open_run(config, push_state, num_examples=len(tasks))
+    # Resumed rollouts are part of this run too.
+    log_episodes(run, finished)
 
     async def on_complete(episode: Episode) -> None:
         episode.record_run(EvalRunInfo(id=config.run.id, name=config.run.name))
         await append_episode(out, episode, write_lock)
+        await asyncio.to_thread(log_episodes, run, [episode])
 
     backend = (
         _in_process(env, config, semaphore, on_complete)
         if env is not None
         else _server(config, config.serve, semaphore, on_complete)
     )
-    async with backend as run_slot:
-        # The display slots: in-process ones are the env's own (it fills their live
-        # traces); a served rollout's is a client-side stand-in its worker never sees.
-        planned = [
-            slot
-            for task, n in plan
-            for slot in (
-                env.slots(task, n)
-                if env is not None
-                else [RunSlot(task) for _ in range(n)]
+    # The run is closed out whatever breaks, backend setup and teardown included.
+    try:
+        async with backend as run_slot:
+            # The display slots: in-process ones are the env's own (it fills their live
+            # traces); a served rollout's is a client-side stand-in its worker never sees.
+            planned = [
+                slot
+                for task, n in plan
+                for slot in (
+                    env.slots(task, n)
+                    if env is not None
+                    else [RunSlot(task) for _ in range(n)]
+                )
+            ]
+            slots = [RunSlot.finished(episode) for episode in finished] + planned
+            display = (
+                dashboard(slots, config, start, push=push_state)
+                if config.rich is not None
+                else contextlib.nullcontext()
             )
-        ]
-        slots = [RunSlot.finished(episode) for episode in finished] + planned
-        push_state = None
-        if config.push and config.rich is not None:
-            from verifiers.v1.utils.platform import PushState
-
-            push_state = PushState()
-        display = (
-            dashboard(slots, config, start, push=push_state)
-            if config.rich is not None
-            else contextlib.nullcontext()
-        )
-        async with display:
-            results = await asyncio.gather(*(run_slot(slot) for slot in planned))
-            episodes = finished + list(results)
-            if (
-                push_state is not None
-            ):  # upload off the event loop so the view keeps refreshing
-                from verifiers.v1.utils.platform import push_traces
-
-                push_state.started = True
-                await asyncio.to_thread(push_traces, episodes, config, push_state)
+            async with display:
+                results = await gather_rollouts(run_slot(slot) for slot in planned)
+                episodes = finished + list(results)
+                # Drain and close out off the event loop so the view keeps refreshing.
+                # Shielded: a Ctrl-C here must not cancel the close-out before the
+                # worker picks it up (a cancelled executor item never runs), so it
+                # runs to completion first and the interrupt is re-raised after —
+                # by which point the run is finished and `abort_run` has nothing to do.
+                await run_shielded(
+                    asyncio.to_thread(finish_run, run, episodes, push_state)
+                )
+    except BaseException as e:
+        await asyncio.to_thread(abort_run, run, e, push_state)
+        raise
     return episodes

{verifiers-0.3.2.dev72 → verifiers-0.3.2.dev74}/verifiers/v1/configs/cli/eval.py

@@ -48,13 +48,29 @@ class RunConfig(BaseConfig):
     """Run directory name — the run writes to `output_dir / dir`. Defaults to `run.name`;
     set it only when the directory should differ from the display name."""
 
-    # TODO: fetch the id from the Prime SDK once runs are registered there.
-    _id: str = PrivateAttr(default_factory=lambda: str(uuid4()))
+    attach: str | None = None
+    """Stream into a run the launcher already created on the platform instead of opening a
+    new one — its evaluation id. Hosted evaluations pass the sandbox's `$EVALUATION_ID`
+    here; the platform owns that run's record and status, and a run that cannot be
+    attached to is an error rather than a local fallback. Requires `push`."""
+
+    _id: str | None = PrivateAttr(default=None)
 
     @property
     def id(self) -> str:
+        """The run's one id, assigned by `open_run` from the prime-runs handle: the
+        platform's evaluation id online, the SDK's local id otherwise. The run mints
+        none of its own, so the run dir, every trace and the dashboard agree."""
+        if self._id is None:
+            raise RuntimeError("the run has no id until `open_run` has opened it")
         return self._id
 
+    def assign_id(self, run_id: str) -> None:
+        """Called once by `open_run`, before the first rollout."""
+        if self._id is not None and self._id != run_id:
+            raise RuntimeError(f"the run already has id {self._id!r}")
+        self._id = run_id
+
 
 class EvalConfig(BaseConfig):
     env: SerializeAsAny[EnvConfig] = SingleAgentEnvConfig()
@@ -135,3 +151,11 @@ class EvalConfig(BaseConfig):
         if self.run.dir is None:
             self.run.dir = self.run.name
         return self
+
+    @model_validator(mode="after")
+    def attach_needs_push(self):
+        if self.run.attach and not self.push:
+            raise ValueError(
+                "run.attach names a run on the platform, so it needs push (drop --no-push)"
+            )
+        return self

verifiers-0.3.2.dev74/verifiers/v1/utils/platform.py

@@ -0,0 +1,178 @@
+"""The eval's run on the Prime Intellect platform (`--no-push` to keep it local)."""
+
+import asyncio
+import logging
+import os
+from collections.abc import Mapping
+from dataclasses import dataclass
+from typing import Any
+
+import prime_runs as pr
+from prime_runs.projection import (
+    build_samples,  # noqa: F401 - prime-rl imports it from here
+)
+
+from verifiers.v1.configs.cli.eval import EvalConfig
+from verifiers.v1.episode import Episode
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class PushState:
+    """The dashboard's view of the run: reads through to it, owns no I/O."""
+
+    run: pr.Run | None = None
+    error: str | None = None
+    incomplete: str | None = None
+    """Set when the run closed out but the uploader lost records on the way: the run
+    exists and holds what did land, so the footer says so rather than "failed"."""
+
+    @property
+    def url(self) -> str | None:
+        return self.run.url if self.run is not None else None
+
+    @property
+    def finished(self) -> bool:
+        return self.run is not None and self.run.finished
+
+    @property
+    def started(self) -> bool:
+        """A live run, or a reason there isn't one."""
+        return self.error is not None or self.url is not None
+
+
+def open_run(config: EvalConfig, state: PushState, *, num_examples: int) -> pr.Run:
+    """Open the run this eval streams into, before the first rollout, and give the
+    config the run's id. A run that cannot be opened is logged and replaced by a
+    disabled one; the eval goes on. With `run.attach` the run already exists on the
+    platform (a hosted evaluation's launcher created it and is waiting on it), so
+    there is no local fallback: failing to attach fails the eval."""
+    attach = config.run.attach
+    identity: dict[str, Any] = {
+        "name": config.run.name,
+        # Resolved by name via the hub's get-or-create; no taskset, nothing to attach to.
+        "environments": [config.env.taskset.id] if config.env.taskset.id else [],
+        "model": config.model,
+        "framework": "verifiers",
+        # The v0 keys the dashboard's lists read. The config itself is a follow-up:
+        # a dump or the launched file can carry credentials and needs masking first.
+        "config": {
+            "model": config.model,
+            "num_examples": num_examples,
+            "rollouts_per_example": config.num_rollouts,
+        },
+    }
+    if config.push and os.getenv(pr.MODE_ENV, "").strip().lower() == "disabled":
+        if attach:
+            raise RuntimeError(
+                f"run.attach={attach!r} names a run on the platform, but "
+                f"{pr.MODE_ENV}=disabled would keep this eval local"
+            )
+        # The SDK's own kill switch; the explicit `mode="online"` below would override it.
+        logger.info("--push: %s=disabled; running without a platform run", pr.MODE_ENV)
+    elif config.push:
+        try:
+            state.run = pr.init(mode="online", id=attach, **identity)
+        except Exception as e:
+            if attach:
+                # The launcher's run would sit at running until it times out; a local
+                # run nobody reads is not a substitute.
+                raise RuntimeError(
+                    f"--run.attach: could not attach to run {attach!r} ({type(e).__name__}: {e})"
+                ) from e
+            logger.warning(
+                "--push: could not open the run (%s: %s); running without it",
+                type(e).__name__,
+                e,
+            )
+            state.error = f"{type(e).__name__}: {e}"
+    if state.run is None:
+        state.run = pr.init(mode="disabled", **identity)
+    # The run's one id: the platform's when online, the SDK's local one otherwise.
+    # The SDK keys every upload to it regardless; this is for the local records.
+    config.run.assign_id(state.run.id)
+    return state.run
+
+
+def log_episodes(run: pr.Run, episodes: list[Episode]) -> None:
+    """Hand finished episodes to the run, best effort: the SDK already keeps upload
+    failures on its own thread, so this only guards the hand-off itself. A problem
+    here is the platform's, never the eval's."""
+    if not episodes:
+        return
+    try:
+        run.log_episodes(episodes)
+    except Exception as e:  # noqa: BLE001 - the rollouts are on disk; report, don't raise
+        logger.warning(
+            "--push: could not queue %d episode(s) (%s: %s)",
+            len(episodes),
+            type(e).__name__,
+            e,
+        )
+
+
+def finish_run(run: pr.Run, episodes: list[Episode], state: PushState) -> None:
+    """Drain, write the run's aggregates, close it out. Blocking: call it off the loop."""
+    try:
+        summary = pr.metrics.from_episodes(episodes)
+    except Exception as e:  # noqa: BLE001 - close the run even without its headline
+        logger.warning(
+            "--push: could not aggregate the run's metrics (%s: %s)",
+            type(e).__name__,
+            e,
+        )
+        summary = None
+    _close(run, state, summary=summary)
+
+
+def abort_run(run: pr.Run, error: BaseException, state: PushState) -> None:
+    """Close the run out after the eval broke, so it doesn't sit at running. Not
+    for a break during `finish_run`: that close-out completes on its own thread
+    and the SDK lets the first `finish()` decide the status — it sets `run.status`
+    as it starts, so an in-flight one is visible here before it is `finished`."""
+    if run.finished or run.status is not pr.RunStatus.RUNNING:
+        return
+    if isinstance(error, (KeyboardInterrupt, asyncio.CancelledError)):
+        status, message = pr.RunStatus.CANCELLED, "interrupted"
+    else:
+        status, message = pr.RunStatus.FAILED, f"{type(error).__name__}: {error}"
+    _close(run, state, status=status, error=message)
+
+
+def _close(
+    run: pr.Run,
+    state: PushState,
+    summary: Mapping[str, Any] | None = None,
+    status: pr.RunStatus = pr.RunStatus.COMPLETED,
+    error: str | None = None,
+) -> None:
+    """`run.finish()`, best effort: the results are on disk, so nothing here may raise."""
+    try:
+        run.finish(summary, status=status, error=error)
+    except Exception as e:  # noqa: BLE001 - the run is over; report, don't raise
+        logger.warning(
+            "--push: could not close out the run (%s: %s)", type(e).__name__, e
+        )
+        if state.error is None:
+            state.error = f"{type(e).__name__}: {e}"
+    else:
+        state.incomplete = _losses(run)
+        if state.incomplete:
+            logger.warning("--push: %s, but %s", status.value, state.incomplete)
+        if run.url:
+            # The run's own status: `finish()` is a no-op once another caller closed it.
+            logger.info("--push: %s -> %s", run.status.value, run.url)
+
+
+def _losses(run: pr.Run) -> str | None:
+    """What the uploader could not store, or `None`. A sink that switched itself off
+    quietly (Prime Traces outside the beta) is not a loss and is not counted here."""
+    parts = [
+        f"{count} record(s) not stored by the {sink} sink"
+        for sink, count in sorted(run.failed_records.items())
+        if count
+    ]
+    if run.dropped_records:
+        parts.append(f"{run.dropped_records} record(s) never queued (uploader overrun)")
+    return "; ".join(parts) or None