verifiers 0.3.2.dev73__tar.gz → 0.3.2.dev75__tar.gz

{verifiers-0.3.2.dev73 → verifiers-0.3.2.dev75}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.3.2.dev73
+Version: 0.3.2.dev75
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers
@@ -35,7 +35,7 @@ Requires-Dist: msgpack>=1.1.2
 Requires-Dist: numpy>=2.1.0
 Requires-Dist: openai<3.0.0,>=2.54.0
 Requires-Dist: prime-pydantic-config[toml]>=0.4.3
-Requires-Dist: prime-runs>=0.1.1
+Requires-Dist: prime-runs>=0.1.2
 Requires-Dist: prime-sandboxes>=0.2.39
 Requires-Dist: prime-tunnel>=0.1.8
 Requires-Dist: pydantic>=2.12.3

{verifiers-0.3.2.dev73 → verifiers-0.3.2.dev75}/pyproject.toml

@@ -34,7 +34,7 @@ dependencies = [
     "openai>=2.54.0,<3.0.0",
     "prime-tunnel>=0.1.8",
     "prime-sandboxes>=0.2.39",
-    "prime-runs>=0.1.1",
+    "prime-runs>=0.1.2",
     "pydantic>=2.12.3",
     "rich>=11.0.0",
     "tenacity>=8.5.0",

verifiers-0.3.2.dev75/tests/v1/test_platform_run.py

@@ -0,0 +1,104 @@
+"""`open_run`: the eval's run on the platform, and `run.attach` for hosted evaluations."""
+
+import prime_runs as pr
+import pytest
+
+from verifiers.v1.configs.cli.eval import EvalConfig
+from verifiers.v1.utils import platform
+from verifiers.v1.utils.platform import PushState, open_run
+
+
+class _FakeRun:
+    """Stands in for `pr.Run`: an id, a URL, nothing else the tests reach for."""
+
+    def __init__(self, run_id: str, mode: str) -> None:
+        self.id = run_id
+        self.mode = mode
+        self.url = (
+            None if mode == "disabled" else f"https://app.example/evaluations/{run_id}"
+        )
+        self.finished = False
+
+
+@pytest.fixture
+def init_calls(monkeypatch):
+    """Record every `pr.init` call; the fake returns a run keyed by the attach id."""
+    calls: list[dict] = []
+
+    def fake_init(**kwargs):
+        calls.append(kwargs)
+        return _FakeRun(kwargs.get("id") or "eval-created", kwargs["mode"])
+
+    monkeypatch.setattr(platform.pr, "init", fake_init)
+    monkeypatch.delenv(pr.MODE_ENV, raising=False)
+    return calls
+
+
+def test_attach_needs_push():
+    with pytest.raises(ValueError, match="needs push"):
+        EvalConfig(run={"attach": "eval-hosted"}, push=False)
+
+
+def test_a_run_without_attach_is_created_on_the_platform(init_calls):
+    config = EvalConfig(push=True)
+    state = PushState()
+
+    run = open_run(config, state, num_examples=3)
+
+    assert init_calls[0]["mode"] == "online"
+    assert init_calls[0]["id"] is None
+    assert run.id == config.run.id == "eval-created"
+
+
+def test_attach_hands_the_launchers_id_to_the_sdk(init_calls):
+    config = EvalConfig(run={"attach": "eval-hosted"})
+    state = PushState()
+
+    run = open_run(config, state, num_examples=3)
+
+    assert init_calls[0]["mode"] == "online"
+    assert init_calls[0]["id"] == "eval-hosted"
+    # The launcher's id is the run's one id: traces and the run dir carry it too.
+    assert run.id == config.run.id == "eval-hosted"
+    assert state.error is None
+
+
+def test_a_failed_attach_fails_the_eval_instead_of_falling_back(monkeypatch):
+    def refuse(**kwargs):
+        raise pr.UnauthorizedError("bad key")
+
+    monkeypatch.setattr(platform.pr, "init", refuse)
+    monkeypatch.delenv(pr.MODE_ENV, raising=False)
+    config = EvalConfig(run={"attach": "eval-hosted"})
+
+    with pytest.raises(RuntimeError, match="could not attach to run 'eval-hosted'"):
+        open_run(config, PushState(), num_examples=3)
+
+
+def test_a_failed_open_without_attach_still_falls_back_to_a_local_run(monkeypatch):
+    calls: list[dict] = []
+
+    def flaky(**kwargs):
+        calls.append(kwargs)
+        if kwargs["mode"] == "online":
+            raise pr.UnauthorizedError("bad key")
+        return _FakeRun("local-run", kwargs["mode"])
+
+    monkeypatch.setattr(platform.pr, "init", flaky)
+    monkeypatch.delenv(pr.MODE_ENV, raising=False)
+    state = PushState()
+
+    run = open_run(EvalConfig(push=True), state, num_examples=3)
+
+    assert [call["mode"] for call in calls] == ["online", "disabled"]
+    assert run.mode == "disabled"
+    assert state.error is not None and "UnauthorizedError" in state.error
+
+
+def test_the_sdk_kill_switch_refuses_an_attach(init_calls, monkeypatch):
+    monkeypatch.setenv(pr.MODE_ENV, "disabled")
+    config = EvalConfig(run={"attach": "eval-hosted"})
+
+    with pytest.raises(RuntimeError, match="PRIME_RUNS_MODE=disabled"):
+        open_run(config, PushState(), num_examples=3)
+    assert init_calls == []

{verifiers-0.3.2.dev73 → verifiers-0.3.2.dev75}/verifiers/v1/configs/cli/eval.py

@@ -48,6 +48,12 @@ class RunConfig(BaseConfig):
     """Run directory name — the run writes to `output_dir / dir`. Defaults to `run.name`;
     set it only when the directory should differ from the display name."""
 
+    attach: str | None = None
+    """Stream into a run the launcher already created on the platform instead of opening a
+    new one — its evaluation id. Hosted evaluations pass the sandbox's `$EVALUATION_ID`
+    here; the platform owns that run's record and status, and a run that cannot be
+    attached to is an error rather than a local fallback. Requires `push`."""
+
     _id: str | None = PrivateAttr(default=None)
 
     @property
@@ -145,3 +151,11 @@ class EvalConfig(BaseConfig):
         if self.run.dir is None:
             self.run.dir = self.run.name
         return self
+
+    @model_validator(mode="after")
+    def attach_needs_push(self):
+        if self.run.attach and not self.push:
+            raise ValueError(
+                "run.attach names a run on the platform, so it needs push (drop --no-push)"
+            )
+        return self

{verifiers-0.3.2.dev73 → verifiers-0.3.2.dev75}/verifiers/v1/utils/platform.py

@@ -45,7 +45,10 @@ class PushState:
 def open_run(config: EvalConfig, state: PushState, *, num_examples: int) -> pr.Run:
     """Open the run this eval streams into, before the first rollout, and give the
     config the run's id. A run that cannot be opened is logged and replaced by a
-    disabled one; the eval goes on."""
+    disabled one; the eval goes on. With `run.attach` the run already exists on the
+    platform (a hosted evaluation's launcher created it and is waiting on it), so
+    there is no local fallback: failing to attach fails the eval."""
+    attach = config.run.attach
     identity: dict[str, Any] = {
         "name": config.run.name,
         # Resolved by name via the hub's get-or-create; no taskset, nothing to attach to.
@@ -61,12 +64,23 @@ def open_run(config: EvalConfig, state: PushState, *, num_examples: int) -> pr.R
         },
     }
     if config.push and os.getenv(pr.MODE_ENV, "").strip().lower() == "disabled":
+        if attach:
+            raise RuntimeError(
+                f"run.attach={attach!r} names a run on the platform, but "
+                f"{pr.MODE_ENV}=disabled would keep this eval local"
+            )
         # The SDK's own kill switch; the explicit `mode="online"` below would override it.
         logger.info("--push: %s=disabled; running without a platform run", pr.MODE_ENV)
     elif config.push:
         try:
-            state.run = pr.init(mode="online", **identity)
-        except Exception as e:  # noqa: BLE001 - a failed upload must not fail the eval
+            state.run = pr.init(mode="online", id=attach, **identity)
+        except Exception as e:
+            if attach:
+                # The launcher's run would sit at running until it times out; a local
+                # run nobody reads is not a substitute.
+                raise RuntimeError(
+                    f"--run.attach: could not attach to run {attach!r} ({type(e).__name__}: {e})"
+                ) from e
             logger.warning(
                 "--push: could not open the run (%s: %s); running without it",
                 type(e).__name__,
@@ -152,13 +166,13 @@ def _close(
 
 
 def _losses(run: pr.Run) -> str | None:
-    """What the uploader could not store, or `None`. A sink that switched itself off
-    quietly (Prime Traces outside the beta) is not a loss and is not counted here."""
-    parts = [
+    """What the run could not finish or store, or `None`."""
+    parts = list(run.errors)
+    parts.extend(
         f"{count} record(s) not stored by the {sink} sink"
         for sink, count in sorted(run.failed_records.items())
         if count
-    ]
+    )
     if run.dropped_records:
         parts.append(f"{run.dropped_records} record(s) never queued (uploader overrun)")
     return "; ".join(parts) or None