verifiers 0.3.2.dev189__tar.gz → 0.3.2.dev191__tar.gz

{verifiers-0.3.2.dev189 → verifiers-0.3.2.dev191}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.3.2.dev189
+Version: 0.3.2.dev191
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

verifiers-0.3.2.dev191/verifiers/v1/cli/eval/resume.py

@@ -0,0 +1,58 @@
+"""Resume an interrupted eval: reload its finished rollouts and run only the rest.
+
+`load` validates saved episodes, applies the environment's completion verdict, and
+uses the shared rollout planner to retain only those matching the current tasks.
+"""
+
+from collections.abc import Callable
+from pathlib import Path
+
+from pydantic_core import from_json
+
+from verifiers.v1.episode import WireEpisode
+from verifiers.v1.task import TaskT
+from verifiers.v1.utils.eval import plan_rollouts
+from verifiers.v1.utils.trace_store import TRACES_FILE
+
+
+def load(
+    resume_dir: Path,
+    tasks: list[TaskT],
+    num_rollouts: int,
+    complete: Callable[[WireEpisode], bool] | None = None,
+) -> list[tuple[TaskT, list[WireEpisode], int]]:
+    """Plan the remaining rollouts and atomically rewrite the stream to kept rows.
+
+    ``complete`` is the keep-verdict (default ``episode.ok``). Torn or malformed
+    rows are discarded; retained rows preserve their original serialized fields.
+    """
+    path = resume_dir / TRACES_FILE
+    verdict = complete if complete is not None else (lambda episode: episode.ok)
+    episodes: list[WireEpisode] = []
+    lines: dict[str, bytes] = {}
+    if path.exists():
+        with path.open("rb") as results:
+            for line in results:
+                if not line.strip():
+                    continue
+                try:
+                    row = from_json(line)
+                    if "traces" not in row:
+                        continue
+                    episode = WireEpisode.model_validate(row)
+                    if not verdict(episode):
+                        continue
+                # A malformed row from any task/episode plugin is owed again.
+                except Exception:  # noqa: BLE001, S112
+                    continue
+                episodes.append(episode)
+                lines.setdefault(
+                    episode.id, line if line.endswith(b"\n") else line + b"\n"
+                )
+    plan = plan_rollouts(tasks, num_rollouts, episodes)
+    tmp = path.with_suffix(".jsonl.tmp")
+    tmp.write_bytes(
+        b"".join(lines[episode.id] for _, kept, _ in plan for episode in kept)
+    )
+    tmp.replace(path)
+    return plan

{verifiers-0.3.2.dev189 → verifiers-0.3.2.dev191}/verifiers/v1/cli/eval/runner.py

@@ -18,12 +18,12 @@ from verifiers.v1.cli.dashboard import dashboard
 from verifiers.v1.cli.eval import resume
 from verifiers.v1.cli.eval.hint import PRIME_RL_HINT
 from verifiers.v1.cli.output import output_path, save_config
-from verifiers.v1.cli.resume import distribute
 from verifiers.v1.clients import ModelContext
 from verifiers.v1.configs.cli.eval import EvalConfig
 from verifiers.v1.env import Env, RunSlot
 from verifiers.v1.episode import Episode, EvalRunInfo
 from verifiers.v1.utils.aio import run_shielded
+from verifiers.v1.utils.eval import plan_rollouts
 from verifiers.v1.utils.platform import (
     PushState,
     abort_run,
@@ -89,40 +89,34 @@ async def run_eval(config: EvalConfig) -> list[Episode]:
         )
     tasks = list(taskset)
     out = output_path(config)
-    # One (task, rollouts-to-run) pair per selected task; resume shrinks the counts.
-    plan = [(task, config.num_rollouts) for task in tasks]
-    # Kept on-disk rollouts rejoin the run as finished episodes; only owed ones re-run.
-    finished: list[Episode] = []
     if config.resume:
-        keys = [task.hash for task in tasks]
         # the env's own keep-verdict decides what resumes
-        loaded, owed = resume.load(
+        plan = resume.load(
             out,
-            keys,
+            tasks,
             config.num_rollouts,
             lambda episode: env.complete(cast(Episode, episode)),
         )
-        finished = [cast(Episode, episode) for episode in loaded]
-        if not owed:  # already complete - report it and exit successfully
+        if not any(n for _, _, n in plan):
             print(
                 f"nothing to resume in {out}: all {len(tasks)}x{config.num_rollouts} "
                 "rollouts already completed without error"
             )
             raise SystemExit(0)
-        counts = distribute(keys, owed, config.num_rollouts)
-        plan = [(task, n) for task, n in zip(tasks, counts) if n]
         logger.info(
             "resuming %s: %d task(s), %d rollout(s) owed",
             out,
-            len(plan),
-            sum(owed.values()),
+            sum(n > 0 for _, _, n in plan),
+            sum(n for _, _, n in plan),
         )
     else:
+        plan = plan_rollouts(tasks, config.num_rollouts)
         save_config(config, out)
         logger.info(
             "running %dx%d rollouts on %s", len(plan), config.num_rollouts, config.model
         )
         logger.info(PRIME_RL_HINT)
+    finished = [cast(Episode, episode) for _, kept, _ in plan for episode in kept]
     start = time.time()
     logger.info("results: %s", out)
 
@@ -148,7 +142,7 @@ async def run_eval(config: EvalConfig) -> list[Episode]:
     try:
         async with _in_process(env, config, semaphore, on_complete) as run_slot:
             # the env's own slots: it fills their live traces as the rollouts run
-            planned = [slot for task, n in plan for slot in env.slots(task, n)]
+            planned = [slot for task, _, n in plan if n for slot in env.slots(task, n)]
             slots = [RunSlot.finished(episode) for episode in finished] + planned
             display = (
                 dashboard(slots, config, start, push=push_state)

{verifiers-0.3.2.dev189 → verifiers-0.3.2.dev191}/verifiers/v1/clients/base.py

@@ -5,7 +5,11 @@ import re
 import httpx
 from openai import AsyncOpenAI
 
-from verifiers.v1.configs.client import BaseClientConfig, resolve_api_key
+from verifiers.v1.configs.client import (
+    BaseClientConfig,
+    resolve_api_key,
+    resolve_headers,
+)
 
 DEFAULT_LIMITS = httpx.Limits(max_connections=1000, max_keepalive_connections=100)
 MAX_RETRIES = 0
@@ -27,7 +31,7 @@ def build_async_openai(config: BaseClientConfig) -> AsyncOpenAI:
     return AsyncOpenAI(
         base_url=config.base_url,
         api_key=resolve_api_key(config),
-        default_headers=config.headers or None,
+        default_headers=resolve_headers(config) or None,
         max_retries=MAX_RETRIES,
         http_client=build_async_httpx(config),
     )

{verifiers-0.3.2.dev189 → verifiers-0.3.2.dev191}/verifiers/v1/clients/eval.py

@@ -9,7 +9,11 @@ from pydantic_core import from_json, to_json
 
 from verifiers.v1.clients.base import build_async_httpx, join_url
 from verifiers.v1.clients.client import SESSION_ID_HEADER, Client, RelayReply
-from verifiers.v1.configs.client import BaseClientConfig, resolve_api_key
+from verifiers.v1.configs.client import (
+    BaseClientConfig,
+    resolve_api_key,
+    resolve_headers,
+)
 from verifiers.v1.dialects import Dialect
 from verifiers.v1.errors import model_error
 from verifiers.v1.graph import PendingTurn
@@ -67,7 +71,7 @@ class EvalClient(Client):
         self.api_key = resolve_api_key(config)
         # Keep endpoint headers separate so they can override intercepted request headers before
         # the dialect's provider authentication is applied.
-        self.headers = dict(config.headers or {})
+        self.headers = resolve_headers(config)
         self.client = build_async_httpx(config)
 
     async def get_response(

{verifiers-0.3.2.dev189 → verifiers-0.3.2.dev191}/verifiers/v1/configs/client.py

@@ -11,6 +11,7 @@ build clients from these. `ClientConfig` is the CLI-selectable discriminated uni
 """
 
 import os
+from itertools import chain
 from typing import Annotated, Literal
 from urllib.parse import urlparse
 
@@ -48,6 +49,9 @@ class BaseClientConfig(BaseConfig):
     api_key_var: str = "PRIME_API_KEY"
     headers: dict[str, str] = Field(default_factory=dict)
     """Extra HTTP headers sent on every request."""
+    headers_from_env: dict[str, str] = Field(default_factory=dict)
+    """Header names mapped to environment variables, resolved when building a client.
+    Set environment values override static headers."""
     timeout: ClientTimeoutConfig = ClientTimeoutConfig()
 
     @model_validator(mode="after")
@@ -65,14 +69,6 @@ class BaseClientConfig(BaseConfig):
             and self.base_url == DEFAULT_PRIME_INFERENCE_URL
         ):
             self.base_url = prime_base_url
-        host = urlparse(self.base_url).hostname or ""
-        if host != PRIME_INFERENCE_HOST and not host.endswith(
-            f".{PRIME_INFERENCE_HOST}"
-        ):
-            return self
-        team_id = os.environ.get("PRIME_TEAM_ID") or prime_config.get("team_id")
-        if team_id:
-            self.headers.setdefault(PRIME_TEAM_ID_HEADER, team_id)
         return self
 
 
@@ -121,3 +117,29 @@ def resolve_api_key(config: BaseClientConfig) -> str:
     ):
         api_key = load_prime_config().get("api_key")
     return api_key or "EMPTY"
+
+
+def resolve_headers(config: BaseClientConfig) -> dict[str, str]:
+    """Resolve request headers for this destination without storing credentials in config."""
+    # Keep original spelling for SDKs that merge default headers case-sensitively.
+    names: dict[str, str] = {}
+    headers: dict[str, str] = {}
+    for name, value in chain(
+        config.headers.items(),
+        (
+            (name, value)
+            for name, variable in config.headers_from_env.items()
+            if (value := os.environ.get(variable)) is not None
+        ),
+    ):
+        headers[names.setdefault(name.lower(), name)] = value
+    host = urlparse(config.base_url).hostname or ""
+    if (
+        config.api_key_var == "PRIME_API_KEY"
+        and (host == PRIME_INFERENCE_HOST or host.endswith(f".{PRIME_INFERENCE_HOST}"))
+        and PRIME_TEAM_ID_HEADER.lower() not in names
+    ):
+        team_id = os.environ.get("PRIME_TEAM_ID") or load_prime_config().get("team_id")
+        if team_id:
+            headers[PRIME_TEAM_ID_HEADER] = team_id
+    return headers

{verifiers-0.3.2.dev189 → verifiers-0.3.2.dev191}/verifiers/v1/gepa/reflection.py

@@ -20,6 +20,8 @@ def build_reflection_lm(config: GEPAConfig) -> Callable[[str], str]:
             base_url=client.base_url,
             api_key_var=client.api_key_var,
             headers=client.headers,
+            headers_from_env=client.headers_from_env,
+            timeout=client.timeout,
         )
     )
 

{verifiers-0.3.2.dev189 → verifiers-0.3.2.dev191}/verifiers/v1/interception/server.py

@@ -45,6 +45,7 @@ from verifiers.v1.configs.client import (
     BaseClientConfig,
     TrainClientConfig,
     resolve_api_key,
+    resolve_headers,
 )
 from verifiers.v1.dialects import DIALECTS, Dialect
 from verifiers.v1.dialects.base import (
@@ -962,7 +963,7 @@ class InterceptionServer(Interception):
         session.adopt(asyncio.current_task())
         logger.debug("intercept models: id=%s", session.trace.id)
         config = session.ctx.client
-        headers = dict(config.headers or {})
+        headers = httpx.Headers(resolve_headers(config))
         headers.update(dialect.auth_headers(resolve_api_key(config)))
         try:
             # Finite read timeout: a hung provider must not stall threshold discovery

verifiers-0.3.2.dev191/verifiers/v1/utils/eval.py

@@ -0,0 +1,36 @@
+"""Rollout planning shared by in-process and served evaluations."""
+
+from collections import defaultdict, deque
+from collections.abc import Iterable
+
+from verifiers.v1.episode import WireEpisode
+from verifiers.v1.task import Task, TaskT
+
+
+def plan_rollouts(
+    tasks: Iterable[TaskT],
+    num_rollouts: int,
+    completed: Iterable[WireEpisode] = (),
+) -> list[tuple[TaskT, list[WireEpisode], int]]:
+    """Assign distinct accepted episodes to selected task occurrences by content hash.
+
+    Each occurrence gets up to ``num_rollouts`` saved episodes and the count still
+    missing. Call separately per evaluation source; callers own acceptance and I/O.
+    """
+    saved: dict[str, deque[WireEpisode]] = defaultdict(deque)
+    seen: set[str] = set()
+    for episode in completed:
+        if episode.id in seen:
+            continue
+        seen.add(episode.id)
+        key = episode.task.hash
+        if key is None:
+            key = Task(episode.task.data).hash
+        saved[key].append(episode)
+
+    plan = []
+    for task in tasks:
+        episodes = saved[task.hash]
+        kept = [episodes.popleft() for _ in range(min(num_rollouts, len(episodes)))]
+        plan.append((task, kept, num_rollouts - len(kept)))
+    return plan

verifiers-0.3.2.dev189/verifiers/v1/cli/eval/resume.py

@@ -1,89 +0,0 @@
-"""Resume an interrupted eval: reload its finished rollouts and run only the rest.
-
-`--resume` re-enters the run dir the resolved config points at and writes back into
-it. `load` keeps the good saved rollouts and re-runs what's owed: missing rollouts
-(never written) and errored ones (dropped and redone).
-
-A saved episode is matched to a selected task by its persisted `episode.task.hash`,
-falling back to hashing `episode.task.data` for older rows without one. Tasks with
-identical data are interchangeable, a task whose data changed since the interrupted
-run re-runs, and nothing depends on `data.idx`.
-"""
-
-from collections import Counter, defaultdict
-from collections.abc import Callable
-from pathlib import Path
-
-from pydantic_core import from_json
-
-from verifiers.v1.episode import WireEpisode
-from verifiers.v1.task import task_key
-from verifiers.v1.utils.trace_store import TRACES_FILE
-
-
-def load(
-    resume_dir: Path,
-    selected_keys: list[str],
-    num_rollouts: int,
-    complete: Callable[[WireEpisode], bool] | None = None,
-) -> tuple[list[WireEpisode], dict[str, int]]:
-    """Load the good saved rollouts and diff them against the run's target: returns
-    (kept episodes, rollouts owed per task key). `selected_keys` is one key per
-    selected task (duplicates allowed — a key selected k times is owed up to
-    `k * num_rollouts`; spread back over the tasks with `distribute`). `complete`
-    is the keep-verdict (default `episode.ok`). Rewrites `traces.jsonl` to the
-    kept rows via a temp file + atomic rename; a torn or malformed row is owed
-    again, never a crash."""
-    path = resume_dir / TRACES_FILE
-    targets = {
-        key: count * num_rollouts for key, count in Counter(selected_keys).items()
-    }
-
-    verdict = complete if complete is not None else (lambda episode: episode.ok)
-    good: dict[str, list[tuple[bytes, WireEpisode]]] = defaultdict(list)
-    if path.exists():
-        with path.open("rb") as results:
-            for line in results:
-                if not line.strip():
-                    continue
-                try:
-                    row = from_json(line)
-                    if "traces" not in row:
-                        continue
-                    task = row["task"]
-                    if not isinstance(task, dict):
-                        continue
-                    key = task.get("hash")
-                    if key is None:
-                        key = task_key(task["data"])
-                except (ValueError, KeyError, IndexError, TypeError):
-                    # A torn final line (the run died mid-write) or a foreign shape
-                    # is not a keepable rollout — it's owed again, never a crash.
-                    continue
-                if not isinstance(key, str):
-                    continue
-                if key not in targets or len(good[key]) >= targets[key]:
-                    continue
-                try:
-                    episode = WireEpisode.model_validate(row)
-                    if not verdict(episode):
-                        continue
-                # A malformed row from any task/episode plugin is owed again.
-                except Exception:  # noqa: BLE001, S112
-                    continue
-                good[key].append(
-                    (line if line.endswith(b"\n") else line + b"\n", episode)
-                )
-    keep: list[bytes] = []
-    episodes: list[WireEpisode] = []
-    owed: dict[str, int] = {}
-    for key, target in targets.items():
-        rows = good.get(key, [])
-        keep.extend(line for line, _ in rows)
-        episodes.extend(episode for _, episode in rows)
-        if missing := target - len(rows):
-            owed[key] = missing
-    tmp = path.with_suffix(".jsonl.tmp")
-    tmp.write_bytes(b"".join(keep))
-    tmp.replace(path)
-    return episodes, owed