verifiers 0.1.15.dev434__tar.gz → 0.1.15.dev436__tar.gz

{verifiers-0.1.15.dev434 → verifiers-0.1.15.dev436}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.1.15.dev434
+Version: 0.1.15.dev436
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.1.15.dev434 → verifiers-0.1.15.dev436}/verifiers/v1/cli/dashboard/eval.py

@@ -13,6 +13,7 @@ overview + progress sit on top, above a rule.
 
 import contextlib
 import time
+from typing import TYPE_CHECKING
 
 from pydantic import BaseModel
 from rich.console import Console, Group
@@ -36,6 +37,9 @@ from verifiers.v1.utils.format import (
 )
 from verifiers.utils.pricing_utils import format_cost_usd
 
+if TYPE_CHECKING:
+    from verifiers.v1.push import PushState
+
 # For sizing pages to the terminal: detects the real terminal height/width each access (the live
 # view writes to the same terminal). Reused so we don't rebuild it every refresh tick.
 _CONSOLE = Console()
@@ -208,6 +212,21 @@ def Overview(config: EvalConfig) -> Table:
     return grid
 
 
+def _push_footer(push: "PushState | None") -> Group | None:
+    """The `--push` status line under the rollouts, shown once the run finishes and the upload
+    begins: dim `Pushing traces...` while it runs, then white `Traces pushed (<url>)` or red
+    `Trace push failed (<err>)`. `None` (no line) until the upload starts and when `--push` is off."""
+    if push is None or not push.started:
+        return None
+    if not push.done:
+        line = Text("Pushing traces...", style="dim")
+    elif push.url:
+        line = Text(f"Traces pushed ({push.url})", style="white", overflow="fold")
+    else:
+        line = Text(f"Trace push failed ({push.error})", style="red", overflow="fold")
+    return Group(Rule(style="dim"), line)
+
+
 def Progress(
     rollouts: list[Rollout],
     start: float,
@@ -576,16 +595,22 @@ def _render(
     start: float,
     pager: Pager,
     finished: list[Trace] | None = None,
+    push: "PushState | None" = None,
 ) -> Group:
     now = time.time()
     warning = _warning(config)
     # `{warning}\n\n{overview}` — the caution sits at the very top, blank line, then the overview.
     header = Group(warning, Text(""), Overview(config)) if warning else Overview(config)
-    # Measure the fixed top (header + progress + rule) so the rollout rows fill the rest of the
-    # screen; page through them (timer, or the left/right arrows) when they'd overflow (rich would
-    # otherwise truncate).
+    # The --push status line appears under the rollouts once the upload starts (None during the run
+    # / when off). Measure the fixed top (header + progress + rule) and the footer so the rollout
+    # rows fill what's left; page through them (timer / arrows) when they'd overflow (else rich
+    # truncates).
+    footer = _push_footer(push)
     top = Group(header, Progress(rollouts, start, finished=finished), Rule(style="dim"))
-    rows_per_page = max(1, _CONSOLE.size.height - len(_CONSOLE.render_lines(top)) - 1)
+    reserved = len(_CONSOLE.render_lines(top))
+    if footer is not None:
+        reserved += len(_CONSOLE.render_lines(footer))
+    rows_per_page = max(1, _CONSOLE.size.height - reserved - 1)
     page_groups, index, count = _paginate(_groups(rollouts), rows_per_page, pager, now)
     progress = Progress(
         rollouts,
@@ -593,12 +618,15 @@ def _render(
         page=(index + 1, count) if count > 1 else None,
         finished=finished,
     )
-    return Group(
+    parts = [
         header,
         progress,
         Rule(style="dim"),
         Rows(page_groups, now, config.harness.runtime.type),
-    )
+    ]
+    if footer is not None:
+        parts.append(footer)
+    return Group(*parts)
 
 
 @contextlib.asynccontextmanager
@@ -607,13 +635,17 @@ async def dashboard(
     config: EvalConfig,
     start: float,
     finished: list[Trace] | None = None,
+    push: "PushState | None" = None,
 ):
     """Refresh the live eval view until the `with` block exits, then a final frame. Left/right
     arrows page through rollout rows when they overflow the screen. On resume, `finished` carries
     the kept on-disk rollouts (reloaded as finished traces) so the counts and scores cover the
-    whole run, not just this session's re-run rollouts."""
+    whole run, not just this session's re-run rollouts. `push` is the shared `--push` status the
+    view renders as a line under the rollouts once the upload starts (dim -> white URL / red error),
+    updated by the caller as the inline upload runs and lands."""
     pager = Pager()
     async with live_view(
-        lambda: _render(rollouts, config, start, pager, finished), on_key=pager.on_key
+        lambda: _render(rollouts, config, start, pager, finished, push),
+        on_key=pager.on_key,
     ):
         yield

{verifiers-0.1.15.dev434 → verifiers-0.1.15.dev436}/verifiers/v1/cli/eval/main.py

@@ -105,6 +105,10 @@ def main(argv: list[str] | None = None) -> None:
     else:  # in-process (default), with or without the live dashboard
         env = vf.Environment(config)
         traces = asyncio.run(run_eval(env, config))
+    if config.push and not rich:
+        from verifiers.v1.push import push_traces
+
+        push_traces(traces, config)
     if not rich:  # --rich is the whole output; otherwise dump each trace as JSON
         for trace in traces:
             print(trace.model_dump_json(indent=2, exclude_none=True))

{verifiers-0.1.15.dev434 → verifiers-0.1.15.dev436}/verifiers/v1/cli/eval/runner.py

@@ -89,8 +89,17 @@ async def run_eval(env: Environment, config: EvalConfig) -> list[Trace]:
             for task in tasks
         ]
         rollouts = [rollout for episode in episodes for rollout in episode.rollouts]
+        # --push in the live dashboard: share a status the dashboard renders as a line under the
+        # rollouts once the run finishes and the upload begins (dim -> green URL / red error), and
+        # do the upload inline below so it resolves on screen. Only the rich path has a dashboard;
+        # every other path uploads (+ logs the URL) from `main`.
+        push_state = None
+        if config.push and config.rich:
+            from verifiers.v1.push import PushState
+
+            push_state = PushState()
         display = (
-            dashboard(rollouts, config, start, finished=finished)
+            dashboard(rollouts, config, start, finished=finished, push=push_state)
             if config.rich
             else contextlib.nullcontext()
         )
@@ -98,7 +107,14 @@ async def run_eval(env: Environment, config: EvalConfig) -> list[Trace]:
             results = await asyncio.gather(
                 *(episode.run(semaphore, on_complete) for episode in episodes)
             )
-    traces = [trace for episode_traces in results for trace in episode_traces]
+            traces = [trace for episode_traces in results for trace in episode_traces]
+            if (
+                push_state is not None
+            ):  # upload off the event loop so the view keeps refreshing
+                from verifiers.v1.push import push_traces
+
+                push_state.started = True
+                await asyncio.to_thread(push_traces, traces, config, push_state)
     await client.close()
     return traces
 

{verifiers-0.1.15.dev434 → verifiers-0.1.15.dev436}/verifiers/v1/clients/eval.py

@@ -16,6 +16,7 @@ from collections.abc import Mapping
 import re
 
 import httpx
+from pydantic import ValidationError
 from pydantic_core import from_json, to_json
 
 from verifiers.v1.clients.client import SESSION_ID_HEADER, Client, RelayReply
@@ -95,8 +96,18 @@ class EvalClient(Client):
             dialect.apply_overrides(body, model, sampling_args),
             self._headers(dialect, headers, session_id),
         )
-        raw = from_json(resp.content)
-        response = dialect.parse_response(dialect.validate_response(raw))
+        # A corrupted response (e.g. an HTML error page or a truncated body on a
+        # flaky tunnel) surfaces as a JSON parse failure or a schema validation
+        # failure — map these to a retryable 502 so the harness SDK retries the
+        # call instead of crashing the whole rollout on one bad response.
+        try:
+            raw = from_json(resp.content)
+            response = dialect.parse_response(dialect.validate_response(raw))
+        except (ValueError, ValidationError) as e:
+            raise model_error(
+                f"malformed upstream response: {type(e).__name__}: {e}",
+                status_code=502,
+            ) from e
         response.raw = raw  # the program gets the provider's bytes back 1:1
         return response
 
@@ -145,6 +156,8 @@ class EvalClient(Client):
             raise model_error(str(e), status_code=504) from e
         except httpx.HTTPError as e:
             raise model_error(str(e), status_code=503) from e
+        except ConnectionResetError as e:
+            raise model_error(str(e), status_code=503) from e
         if not stream:
             try:
                 response.raise_for_status()

{verifiers-0.1.15.dev434 → verifiers-0.1.15.dev436}/verifiers/v1/configs/eval.py

@@ -54,6 +54,10 @@ class EvalConfig(EnvServerConfig):
     server: bool = False
     """Drive rollouts through the env-server worker pool (sized by `--pool.*`) instead of
     in-process — the path prime-rl trains through. Incompatible with `--rich`."""
+    push: bool = True
+    """Upload the finished run to the Prime Intellect platform (the private Evaluations
+    tab) at the end of the eval. On by default; disable with `--no-push`. Needs
+    `$PRIME_API_KEY` or `prime login`."""
     output_dir: Path | None = Field(
         None, validation_alias=AliasChoices("output_dir", "o")
     )

verifiers-0.1.15.dev436/verifiers/v1/push.py

@@ -0,0 +1,209 @@
+"""Push a finished eval run to the Prime Intellect platform (`uv run eval`, `--no-push` to skip).
+
+On by default. Converts each in-memory v1 `Trace` to the platform's (v0) eval-sample schema
+(`trace_to_sample`) and uploads the run over the `/evaluations/` API (create -> push samples ->
+finalize) — the same contract
+`prime eval push` uploads a saved run through, done inline at the end of a run rather than
+later from disk. Auth + base URL come from `$PRIME_API_KEY` / `~/.prime/config.json`
+(written by `prime login`), like the rest of the CLI.
+"""
+
+import logging
+import os
+from dataclasses import dataclass
+from typing import Any
+
+import httpx
+
+from verifiers.utils.client_utils import load_prime_config
+from verifiers.v1.configs.eval import EvalConfig
+from verifiers.v1.trace import Trace
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_API_URL = "https://api.primeintellect.ai"
+DEFAULT_FRONTEND_URL = "https://app.primeintellect.ai"
+
+
+@dataclass
+class PushState:
+    """Live status of the push, shared with the v1 `--rich` dashboard so it can show a status line
+    under the rollouts once the run finishes and the upload begins: dim while uploading, then the
+    viewer URL (green) or the error (red). `started` flips true when the upload begins (the caller
+    sets it), `done` when it returns; `url` is set on success and `error` on a skip/failure. No line
+    is shown until `started`. Populated by `push_traces` (pass the state in)."""
+
+    started: bool = False
+    done: bool = False
+    url: str | None = None
+    error: str | None = None
+
+
+def trace_to_sample(trace: Trace, rollout_number: int = 1) -> dict[str, Any]:
+    """One rollout -> the platform's sample dict (the "old" v0 eval-sample format).
+
+    The conversation is the unit — no prompt/completion split (meaningless mid-branch):
+    `completion` is the final branch's messages and `trajectory` carries one message list per
+    branch. `rollout_number` is this rollout's 1-based index within its task's group (callers that
+    don't group rollouts can leave it at the default)."""
+
+    def dump(messages):
+        return [m.model_dump(mode="json", exclude_none=True) for m in messages]
+
+    task = trace.task.model_dump(mode="json", exclude_none=True)
+    branches = trace.branches
+    sample = {
+        "sample_id": trace.id,
+        "example_id": trace.task.idx,
+        "rollout_number": rollout_number,
+        "task": task,
+        "prompt": [],
+        "completion": dump(branches[-1].messages) if branches else [],
+        "answer": task.get("answer"),
+        "reward": trace.reward,
+        "timing": trace.timing.model_dump(mode="json", exclude_none=True),
+        "is_completed": trace.is_completed,
+        "is_truncated": trace.is_truncated,
+        "metrics": trace.metrics,
+        "error": trace.error.model_dump(mode="json", exclude_none=True)
+        if trace.error
+        else None,
+        "stop_condition": trace.stop_condition,
+        "trajectory": [
+            {
+                "messages": dump(branch.messages),
+                "num_input_tokens": branch.num_input_tokens,
+                "num_output_tokens": branch.num_output_tokens,
+            }
+            for branch in branches
+        ],
+        "token_usage": trace.usage.model_dump(mode="json", exclude_none=True)
+        if trace.usage
+        else None,
+        "info": dict(trace.info) or None,
+    }
+    # Flatten each sub-reward onto the sample as a top-level key, the way v0's `state_to_output`
+    # does. Env metrics stay in the nested `metrics` field.
+    for name, value in trace.rewards.items():
+        sample.setdefault(name, value)
+    return sample
+
+
+def _creds() -> tuple[str | None, str, str, str | None]:
+    """(api_key, api_base, frontend_url, team_id) from env vars / `~/.prime/config.json`."""
+    cfg = load_prime_config()
+    api_key = os.getenv("PRIME_API_KEY") or cfg.get("api_key")
+    base = (
+        os.getenv("PRIME_API_BASE_URL")
+        or os.getenv("PRIME_BASE_URL")
+        or cfg.get("base_url")
+        or DEFAULT_API_URL
+    )
+    base = base.rstrip("/").removesuffix("/api/v1")
+    frontend = (
+        os.getenv("PRIME_FRONTEND_URL")
+        or cfg.get("frontend_url")
+        or DEFAULT_FRONTEND_URL
+    )
+    team_id = os.getenv("PRIME_TEAM_ID") or cfg.get("team_id")
+    return api_key, base, frontend, team_id
+
+
+def push_traces(
+    traces: list[Trace], config: EvalConfig, state: "PushState | None" = None
+) -> str | None:
+    """Upload a finished run to the platform; return the viewer URL (None if skipped/failed).
+
+    Resolves the env (get-or-create) by name so a local run uploads without a prior
+    `prime env push`, then create evaluation -> push samples -> finalize. When `state` is given
+    (the v1 `--rich` path), record the outcome on it (`url` on success, `error` on skip/failure,
+    `done` when finished) so the dashboard's status line resolves."""
+
+    def finish(url: str | None = None, error: str | None = None) -> str | None:
+        if state is not None:
+            state.url = url
+            state.error = error
+            state.done = True
+        return url
+
+    api_key, base, frontend, team_id = _creds()
+    if not api_key:
+        logger.warning(
+            "--push: no PRIME_API_KEY (set it or run `prime login`); skipping upload"
+        )
+        return finish(error="no PRIME_API_KEY (run `prime login`)")
+
+    def compute_metrics() -> dict[str, Any]:
+        """Run-level aggregates as v0's `GenerateMetadata`: `avg_reward` (mean over all traces),
+        `avg_metrics` (each sub-reward and env-metric averaged over the traces that recorded it),
+        and `avg_error` (errored fraction) — what the overview renders."""
+        sums: dict[str, float] = {}
+        counts: dict[str, int] = {}
+        for trace in traces:
+            for name, value in {**trace.rewards, **trace.metrics}.items():
+                sums[name] = sums.get(name, 0.0) + value
+                counts[name] = counts.get(name, 0) + 1
+        n = len(traces)
+        return {
+            "avg_reward": sum(t.reward for t in traces) / n if n else 0.0,
+            "avg_metrics": {name: sums[name] / counts[name] for name in sums},
+            "avg_error": sum(t.has_error for t in traces) / n if n else 0.0,
+        }
+
+    env_name = config.taskset.id or config.id
+    metrics = compute_metrics()
+    counts: dict[int, int] = {}
+    samples = []
+    for trace in traces:
+        counts[trace.task.idx] = counts.get(trace.task.idx, 0) + 1
+        samples.append(trace_to_sample(trace, counts[trace.task.idx]))
+
+    metadata = {
+        "framework": "verifiers",
+        "run_id": config.uuid,
+        "model": config.model,
+        "num_examples": len(counts),
+        "rollouts_per_example": config.num_rollouts,
+        **metrics,
+    }
+
+    team = {"team_id": team_id} if team_id else {}
+    api = f"{base}/api/v1"
+    headers = {"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}
+    # The run is already done and its results saved; a network blip or platform error here must
+    # not crash the run (or, on the non-rich path, swallow the final per-trace output that prints
+    # after this) — log and skip the upload instead.
+    try:
+        with httpx.Client(headers=headers, timeout=300.0) as client:
+
+            def post(path: str, body: dict) -> dict:
+                resp = client.post(f"{api}{path}", json=body)
+                resp.raise_for_status()
+                return resp.json()
+
+            env_id = post("/environmentshub/resolve", {"name": env_name, **team})[
+                "data"
+            ]["id"]
+            eval_id = post(
+                "/evaluations/",
+                {
+                    "name": f"{env_name}--{config.model}--{config.uuid[:8]}",
+                    "environments": [{"id": env_id}],
+                    "model_name": config.model,
+                    "dataset": env_name,
+                    "framework": "verifiers",
+                    "metadata": metadata,
+                    "metrics": metrics,
+                    "tags": [],
+                    **team,
+                },
+            )["evaluation_id"]
+            post(f"/evaluations/{eval_id}/samples", {"samples": samples})
+            post(f"/evaluations/{eval_id}/finalize", {"metrics": metrics})
+    except Exception as e:
+        logger.warning("--push: upload failed (%s: %s); skipping", type(e).__name__, e)
+        return finish(error=f"{type(e).__name__}: {e}")
+
+    url = f"{frontend}/dashboard/evaluations/{eval_id}"
+    logger.info("--push: uploaded %d samples -> %s", len(samples), url)
+    return finish(url=url)