verifiers 0.3.2.dev72__tar.gz → 0.3.2.dev73__tar.gz

{verifiers-0.3.2.dev72 → verifiers-0.3.2.dev73}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.3.2.dev72
+Version: 0.3.2.dev73
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers
@@ -35,6 +35,7 @@ Requires-Dist: msgpack>=1.1.2
 Requires-Dist: numpy>=2.1.0
 Requires-Dist: openai<3.0.0,>=2.54.0
 Requires-Dist: prime-pydantic-config[toml]>=0.4.3
+Requires-Dist: prime-runs>=0.1.1
 Requires-Dist: prime-sandboxes>=0.2.39
 Requires-Dist: prime-tunnel>=0.1.8
 Requires-Dist: pydantic>=2.12.3

{verifiers-0.3.2.dev72 → verifiers-0.3.2.dev73}/pyproject.toml

@@ -34,6 +34,7 @@ dependencies = [
     "openai>=2.54.0,<3.0.0",
     "prime-tunnel>=0.1.8",
     "prime-sandboxes>=0.2.39",
+    "prime-runs>=0.1.1",
     "pydantic>=2.12.3",
     "rich>=11.0.0",
     "tenacity>=8.5.0",
@@ -146,6 +147,7 @@ ty = "2026-07-28T00:00:00Z"
 # PrimeIntellect-published on PyPI (trusted publisher)
 prime-tunnel = false
 prime-sandboxes = false
+prime-runs = false
 prime-pydantic-config = false
 renderers = false
 

{verifiers-0.3.2.dev72 → verifiers-0.3.2.dev73}/tests/v1/conftest.py

@@ -188,6 +188,7 @@ def _eval_config(
         output_dir=output_dir.parent,
         run={"dir": output_dir.name},
         model=CI_MODEL,
+        push=False,
     )
 
 

{verifiers-0.3.2.dev72 → verifiers-0.3.2.dev73}/verifiers/v1/cli/dashboard/eval.py

@@ -281,17 +281,25 @@ def Overview(config: EvalConfig) -> Table:
 
 
 def _push_footer(push: "PushState | None") -> Group | None:
-    """The `--push` status line under the rollouts, shown once the run finishes and the upload
-    begins: dim `Pushing traces...` while it runs, then white `Traces pushed (<url>)` or red
-    `Trace push failed (<err>)`. `None` (no line) until the upload starts and when `--push` is off."""
+    """The `--push` line under the rollouts: dim with the run's URL while it streams,
+    white once pushed, red when it failed. `None` when `--push` is off or the run stayed
+    local."""
     if push is None or not push.started:
         return None
-    if not push.done:
-        line = Text("Pushing traces...", style="dim")
-    elif push.url:
+    if push.error and push.url:
+        # The run exists and holds what streamed up; only closing it out failed.
         line = Text(f"Traces pushed ({push.url})", style="white", overflow="fold")
-    else:
+        line.append(f"  not closed out: {push.error}", style="red")
+    elif push.error:
         line = Text(f"Trace push failed ({push.error})", style="red", overflow="fold")
+    elif push.incomplete and push.url:
+        # Closed out, but the uploader lost records: what landed is there, say what didn't.
+        line = Text(f"Traces pushed ({push.url})", style="white", overflow="fold")
+        line.append(f"  incomplete: {push.incomplete}", style="red")
+    elif not push.finished:
+        line = Text(f"Pushing traces ({push.url})", style="dim", overflow="fold")
+    else:
+        line = Text(f"Traces pushed ({push.url})", style="white", overflow="fold")
     return Group(Rule(style="dim"), line)
 
 

{verifiers-0.3.2.dev72 → verifiers-0.3.2.dev73}/verifiers/v1/cli/eval/main.py

@@ -134,10 +134,6 @@ def main(argv: list[str] | None = None) -> None:
         # Graceful cleanup has already run (each rollout's `finally`); partial results are on
         # disk. Exit on the conventional Ctrl-C code without a traceback.
         raise SystemExit(130)
-    if config.push and config.rich is None:
-        from verifiers.v1.utils.platform import push_traces
-
-        push_traces(episodes, config)
     if (
         config.rich is None
     ):  # --rich is the whole output; otherwise dump each trace as JSON

{verifiers-0.3.2.dev72 → verifiers-0.3.2.dev73}/verifiers/v1/cli/eval/runner.py

@@ -13,8 +13,8 @@ import asyncio
 import contextlib
 import logging
 import time
-from collections.abc import AsyncIterator, Awaitable, Callable
-from typing import cast
+from collections.abc import AsyncIterator, Awaitable, Callable, Iterable
+from typing import TypeVar, cast
 
 from verifiers.v1.cli.dashboard import dashboard
 from verifiers.v1.cli.eval import resume
@@ -30,9 +30,36 @@ from verifiers.v1.configs.cli.eval import EvalConfig
 from verifiers.v1.configs.serve import ServeConfig
 from verifiers.v1.env import Env, RunSlot
 from verifiers.v1.episode import Episode, EvalRunInfo
+from verifiers.v1.utils.aio import run_shielded
+from verifiers.v1.utils.platform import (
+    PushState,
+    abort_run,
+    finish_run,
+    log_episodes,
+    open_run,
+)
 
 logger = logging.getLogger(__name__)
 
+T = TypeVar("T")
+
+
+async def gather_rollouts(rollouts: Iterable[Awaitable[T]]) -> list[T]:
+    """`asyncio.gather`, but one rollout failing cancels the rest and waits for them
+    to unwind, so nothing keeps uploading into a run the caller is already closing.
+    Not a `TaskGroup`: that wraps errors in an `ExceptionGroup`, and `main` would no
+    longer see a `KeyboardInterrupt` as Ctrl-C."""
+    tasks = [asyncio.ensure_future(rollout) for rollout in rollouts]
+    try:
+        return await asyncio.gather(*tasks)
+    except BaseException:
+        for task in tasks:
+            task.cancel()
+        # return_exceptions: wait for every task, not just the first to cancel.
+        await asyncio.gather(*tasks, return_exceptions=True)
+        raise
+
+
 RunSlotFn = Callable[[RunSlot], Awaitable[Episode]]
 OnComplete = Callable[[Episode], Awaitable[None]]
 
@@ -203,47 +230,55 @@ async def run_eval(config: EvalConfig) -> list[Episode]:
         asyncio.Semaphore(config.max_concurrent) if config.max_concurrent else None
     )
     write_lock = asyncio.Lock()
+    push_state = PushState()
+
+    # Opened before the first rollout so every episode streams as it lands.
+    run = open_run(config, push_state, num_examples=len(tasks))
+    # Resumed rollouts are part of this run too.
+    log_episodes(run, finished)
 
     async def on_complete(episode: Episode) -> None:
         episode.record_run(EvalRunInfo(id=config.run.id, name=config.run.name))
         await append_episode(out, episode, write_lock)
+        await asyncio.to_thread(log_episodes, run, [episode])
 
     backend = (
         _in_process(env, config, semaphore, on_complete)
         if env is not None
         else _server(config, config.serve, semaphore, on_complete)
     )
-    async with backend as run_slot:
-        # The display slots: in-process ones are the env's own (it fills their live
-        # traces); a served rollout's is a client-side stand-in its worker never sees.
-        planned = [
-            slot
-            for task, n in plan
-            for slot in (
-                env.slots(task, n)
-                if env is not None
-                else [RunSlot(task) for _ in range(n)]
+    # The run is closed out whatever breaks, backend setup and teardown included.
+    try:
+        async with backend as run_slot:
+            # The display slots: in-process ones are the env's own (it fills their live
+            # traces); a served rollout's is a client-side stand-in its worker never sees.
+            planned = [
+                slot
+                for task, n in plan
+                for slot in (
+                    env.slots(task, n)
+                    if env is not None
+                    else [RunSlot(task) for _ in range(n)]
+                )
+            ]
+            slots = [RunSlot.finished(episode) for episode in finished] + planned
+            display = (
+                dashboard(slots, config, start, push=push_state)
+                if config.rich is not None
+                else contextlib.nullcontext()
             )
-        ]
-        slots = [RunSlot.finished(episode) for episode in finished] + planned
-        push_state = None
-        if config.push and config.rich is not None:
-            from verifiers.v1.utils.platform import PushState
-
-            push_state = PushState()
-        display = (
-            dashboard(slots, config, start, push=push_state)
-            if config.rich is not None
-            else contextlib.nullcontext()
-        )
-        async with display:
-            results = await asyncio.gather(*(run_slot(slot) for slot in planned))
-            episodes = finished + list(results)
-            if (
-                push_state is not None
-            ):  # upload off the event loop so the view keeps refreshing
-                from verifiers.v1.utils.platform import push_traces
-
-                push_state.started = True
-                await asyncio.to_thread(push_traces, episodes, config, push_state)
+            async with display:
+                results = await gather_rollouts(run_slot(slot) for slot in planned)
+                episodes = finished + list(results)
+                # Drain and close out off the event loop so the view keeps refreshing.
+                # Shielded: a Ctrl-C here must not cancel the close-out before the
+                # worker picks it up (a cancelled executor item never runs), so it
+                # runs to completion first and the interrupt is re-raised after —
+                # by which point the run is finished and `abort_run` has nothing to do.
+                await run_shielded(
+                    asyncio.to_thread(finish_run, run, episodes, push_state)
+                )
+    except BaseException as e:
+        await asyncio.to_thread(abort_run, run, e, push_state)
+        raise
     return episodes

{verifiers-0.3.2.dev72 → verifiers-0.3.2.dev73}/verifiers/v1/configs/cli/eval.py

@@ -48,13 +48,23 @@ class RunConfig(BaseConfig):
     """Run directory name — the run writes to `output_dir / dir`. Defaults to `run.name`;
     set it only when the directory should differ from the display name."""
 
-    # TODO: fetch the id from the Prime SDK once runs are registered there.
-    _id: str = PrivateAttr(default_factory=lambda: str(uuid4()))
+    _id: str | None = PrivateAttr(default=None)
 
     @property
     def id(self) -> str:
+        """The run's one id, assigned by `open_run` from the prime-runs handle: the
+        platform's evaluation id online, the SDK's local id otherwise. The run mints
+        none of its own, so the run dir, every trace and the dashboard agree."""
+        if self._id is None:
+            raise RuntimeError("the run has no id until `open_run` has opened it")
         return self._id
 
+    def assign_id(self, run_id: str) -> None:
+        """Called once by `open_run`, before the first rollout."""
+        if self._id is not None and self._id != run_id:
+            raise RuntimeError(f"the run already has id {self._id!r}")
+        self._id = run_id
+
 
 class EvalConfig(BaseConfig):
     env: SerializeAsAny[EnvConfig] = SingleAgentEnvConfig()

verifiers-0.3.2.dev73/verifiers/v1/utils/platform.py

@@ -0,0 +1,164 @@
+"""The eval's run on the Prime Intellect platform (`--no-push` to keep it local)."""
+
+import asyncio
+import logging
+import os
+from collections.abc import Mapping
+from dataclasses import dataclass
+from typing import Any
+
+import prime_runs as pr
+from prime_runs.projection import (
+    build_samples,  # noqa: F401 - prime-rl imports it from here
+)
+
+from verifiers.v1.configs.cli.eval import EvalConfig
+from verifiers.v1.episode import Episode
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class PushState:
+    """The dashboard's view of the run: reads through to it, owns no I/O."""
+
+    run: pr.Run | None = None
+    error: str | None = None
+    incomplete: str | None = None
+    """Set when the run closed out but the uploader lost records on the way: the run
+    exists and holds what did land, so the footer says so rather than "failed"."""
+
+    @property
+    def url(self) -> str | None:
+        return self.run.url if self.run is not None else None
+
+    @property
+    def finished(self) -> bool:
+        return self.run is not None and self.run.finished
+
+    @property
+    def started(self) -> bool:
+        """A live run, or a reason there isn't one."""
+        return self.error is not None or self.url is not None
+
+
+def open_run(config: EvalConfig, state: PushState, *, num_examples: int) -> pr.Run:
+    """Open the run this eval streams into, before the first rollout, and give the
+    config the run's id. A run that cannot be opened is logged and replaced by a
+    disabled one; the eval goes on."""
+    identity: dict[str, Any] = {
+        "name": config.run.name,
+        # Resolved by name via the hub's get-or-create; no taskset, nothing to attach to.
+        "environments": [config.env.taskset.id] if config.env.taskset.id else [],
+        "model": config.model,
+        "framework": "verifiers",
+        # The v0 keys the dashboard's lists read. The config itself is a follow-up:
+        # a dump or the launched file can carry credentials and needs masking first.
+        "config": {
+            "model": config.model,
+            "num_examples": num_examples,
+            "rollouts_per_example": config.num_rollouts,
+        },
+    }
+    if config.push and os.getenv(pr.MODE_ENV, "").strip().lower() == "disabled":
+        # The SDK's own kill switch; the explicit `mode="online"` below would override it.
+        logger.info("--push: %s=disabled; running without a platform run", pr.MODE_ENV)
+    elif config.push:
+        try:
+            state.run = pr.init(mode="online", **identity)
+        except Exception as e:  # noqa: BLE001 - a failed upload must not fail the eval
+            logger.warning(
+                "--push: could not open the run (%s: %s); running without it",
+                type(e).__name__,
+                e,
+            )
+            state.error = f"{type(e).__name__}: {e}"
+    if state.run is None:
+        state.run = pr.init(mode="disabled", **identity)
+    # The run's one id: the platform's when online, the SDK's local one otherwise.
+    # The SDK keys every upload to it regardless; this is for the local records.
+    config.run.assign_id(state.run.id)
+    return state.run
+
+
+def log_episodes(run: pr.Run, episodes: list[Episode]) -> None:
+    """Hand finished episodes to the run, best effort: the SDK already keeps upload
+    failures on its own thread, so this only guards the hand-off itself. A problem
+    here is the platform's, never the eval's."""
+    if not episodes:
+        return
+    try:
+        run.log_episodes(episodes)
+    except Exception as e:  # noqa: BLE001 - the rollouts are on disk; report, don't raise
+        logger.warning(
+            "--push: could not queue %d episode(s) (%s: %s)",
+            len(episodes),
+            type(e).__name__,
+            e,
+        )
+
+
+def finish_run(run: pr.Run, episodes: list[Episode], state: PushState) -> None:
+    """Drain, write the run's aggregates, close it out. Blocking: call it off the loop."""
+    try:
+        summary = pr.metrics.from_episodes(episodes)
+    except Exception as e:  # noqa: BLE001 - close the run even without its headline
+        logger.warning(
+            "--push: could not aggregate the run's metrics (%s: %s)",
+            type(e).__name__,
+            e,
+        )
+        summary = None
+    _close(run, state, summary=summary)
+
+
+def abort_run(run: pr.Run, error: BaseException, state: PushState) -> None:
+    """Close the run out after the eval broke, so it doesn't sit at running. Not
+    for a break during `finish_run`: that close-out completes on its own thread
+    and the SDK lets the first `finish()` decide the status — it sets `run.status`
+    as it starts, so an in-flight one is visible here before it is `finished`."""
+    if run.finished or run.status is not pr.RunStatus.RUNNING:
+        return
+    if isinstance(error, (KeyboardInterrupt, asyncio.CancelledError)):
+        status, message = pr.RunStatus.CANCELLED, "interrupted"
+    else:
+        status, message = pr.RunStatus.FAILED, f"{type(error).__name__}: {error}"
+    _close(run, state, status=status, error=message)
+
+
+def _close(
+    run: pr.Run,
+    state: PushState,
+    summary: Mapping[str, Any] | None = None,
+    status: pr.RunStatus = pr.RunStatus.COMPLETED,
+    error: str | None = None,
+) -> None:
+    """`run.finish()`, best effort: the results are on disk, so nothing here may raise."""
+    try:
+        run.finish(summary, status=status, error=error)
+    except Exception as e:  # noqa: BLE001 - the run is over; report, don't raise
+        logger.warning(
+            "--push: could not close out the run (%s: %s)", type(e).__name__, e
+        )
+        if state.error is None:
+            state.error = f"{type(e).__name__}: {e}"
+    else:
+        state.incomplete = _losses(run)
+        if state.incomplete:
+            logger.warning("--push: %s, but %s", status.value, state.incomplete)
+        if run.url:
+            # The run's own status: `finish()` is a no-op once another caller closed it.
+            logger.info("--push: %s -> %s", run.status.value, run.url)
+
+
+def _losses(run: pr.Run) -> str | None:
+    """What the uploader could not store, or `None`. A sink that switched itself off
+    quietly (Prime Traces outside the beta) is not a loss and is not counted here."""
+    parts = [
+        f"{count} record(s) not stored by the {sink} sink"
+        for sink, count in sorted(run.failed_records.items())
+        if count
+    ]
+    if run.dropped_records:
+        parts.append(f"{run.dropped_records} record(s) never queued (uploader overrun)")
+    return "; ".join(parts) or None