verifiers 0.2.2.dev9__tar.gz → 0.2.2.dev11__tar.gz

{verifiers-0.2.2.dev9 → verifiers-0.2.2.dev11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.2.dev9
+Version: 0.2.2.dev11
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.2.2.dev9 → verifiers-0.2.2.dev11}/tests/v1/fixtures/duet_v1.py

@@ -3,7 +3,7 @@
 The multi-agent fixture for the v1 e2e suite (resolved by id `duet-v1`): an
 `Env` subclass exported alongside its taskset, with two roles ("a", a trainable
 seat on the run's model; "b", pinned untrainable), a `run()` that fans both out on
-the task, and a `finalize()` that records a sibling-dependent signal. One eval rollout
+the task, and a `finalize()` that records a sibling-dependent signal. One episode
 should land one record carrying two role-stamped traces.
 """
 

{verifiers-0.2.2.dev9 → verifiers-0.2.2.dev11}/tests/v1/test_e2e.py

@@ -265,8 +265,8 @@ async def test_agentic(run_v1, harness, harness_runtime, tmp_path):
 @pytest.mark.e2e
 async def test_multi_agent_env(run_v1, tmp_path):
     """An `Env` subclass shipped with its taskset (duet-v1): two roles run the
-    task, `score()` episodes a sibling-dependent metric, and one eval rollout lands one
-    episode carrying two role-stamped traces."""
+    task, `score()` episodes a sibling-dependent metric, and one episode carries
+    two role-stamped traces."""
     import json
 
     traces = await run_v1(
@@ -275,7 +275,7 @@ async def test_multi_agent_env(run_v1, tmp_path):
         output_dir=tmp_path,
         max_turns=2,
     )
-    assert len(traces) == 2  # one env-rollout, one trace per role
+    assert len(traces) == 2  # one episode, one trace per role
     assert sorted(t.agent_name for t in traces) == ["a", "b"]
     (b,) = [t for t in traces if t.agent_name == "b"]
     assert b.trainable is False
@@ -305,7 +305,7 @@ async def test_env_id_best_of_n(run_v1, tmp_path):
         output_dir=tmp_path,
         max_turns=2,
     )
-    assert len(traces) == 2  # one env-rollout, two attempts
+    assert len(traces) == 2  # one episode, two attempts
     assert all(t.agent_name == "agent" and t.ok for t in traces)
     assert any(t.metrics["best"] == 1.0 for t in traces)
     assert all(t.metrics["pass_at_n"] == 1.0 for t in traces)  # echo always passes

{verifiers-0.2.2.dev9 → verifiers-0.2.2.dev11}/verifiers/v1/agent.py

@@ -350,7 +350,7 @@ class Agent:
 
 
 class _EpisodeAgent(Agent):
-    """One role's `Agent` for one env-rollout, built fresh per episode (a cheap
+    """One role's `Agent` for one episode, built fresh each time (a cheap
     bundle of references — expensive resources are env-owned and borrowed, so no
     state spans concurrent episodes): traces get their agent standing the moment
     they're created, finished ones land in `completed` (the episode's traces),

{verifiers-0.2.2.dev9 → verifiers-0.2.2.dev11}/verifiers/v1/cli/dashboard/eval.py

@@ -291,7 +291,7 @@ def Progress(
 ) -> Group:
     # On resume, `slots` includes the previous session's kept rollouts (as finished slots), so
     # progress, reward, err, and the breakdown cover the whole run, not just this session's.
-    done = [s for s in slots if s.done]  # fully scored env-rollouts
+    done = [s for s in slots if s.done]  # fully scored episodes
     done_traces = [t for s in done for t in s.traces]
     # Score aggregates read the policy's traces: auxiliary roles (a judge's verdict
     # run, a modeled user) are `trainable=False` and carry no rewards, so counting
@@ -300,7 +300,7 @@ def Progress(
     scored = [t for t in done_traces if t.trainable] or done_traces
     total = len(slots)
     # Headline reward = mean over non-errored traces; when any errored, `format_mean` appends
-    # the global avg (errored count as 0) in parens. `err` is the share of env-rollouts that
+    # the global avg (errored count as 0) in parens. `err` is the share of episodes that
     # ended not-ok (a trace errored, or the env's rollout()/score() hook itself failed).
     reward = format_mean(scored, lambda t: t.reward)
     err = (
@@ -477,7 +477,7 @@ def _stage(trace: Trace) -> str:
     """The stage a live (not-yet-done) rollout is in, derived from its trace's timing
     spans — the engine opens and closes each span exactly at the stage transitions, so
     the current stage is the latest span started but not yet ended. A completed trace
-    whose slot isn't done is waiting on its env-rollout's other traces (and the env's
+    whose slot isn't done is waiting on its episode's other traces (and the env's
     `score()`) — that's scoring."""
     if trace.is_completed:
         return "scoring"
@@ -531,7 +531,7 @@ def _brace(i: int, size: int) -> str:
 
 def Rows(groups: list[list[RunSlot]], now: float, runtime_type: str) -> Table:
     # (brace, state, left sections, result, time); a slot contributes one row per live
-    # trace (a multi-agent env-rollout shows each role's trace), braced per task.
+    # trace (a multi-agent episode shows each role's trace), braced per task.
     rows: list[tuple[str, str, list[str], str, str]] = []
     for group in groups:
         group_rows: list[tuple[str, list[str], str, str]] = []
@@ -572,7 +572,7 @@ def Rows(groups: list[list[RunSlot]], now: float, runtime_type: str) -> Table:
                         ):  # flag a clipped rollout next to its stop condition
                             stop = f"{stop} (truncated)".strip()
                 elif t.is_completed and (err := t.error) is not None:
-                    # An errored trace whose env-rollout is still running its other
+                    # An errored trace whose episode is still running its other
                     # traces (or `score()`) is already a failure — show it, don't
                     # let it sit as "scoring" until the whole episode lands.
                     state, result, stop = "error", err.type, ""

{verifiers-0.2.2.dev9 → verifiers-0.2.2.dev11}/verifiers/v1/cli/eval/runner.py

@@ -1,4 +1,4 @@
-"""The eval runner: fan env-rollouts out with bounded concurrency."""
+"""The eval runner: fan episodes out with bounded concurrency."""
 
 import asyncio
 import contextlib

{verifiers-0.2.2.dev9 → verifiers-0.2.2.dev11}/verifiers/v1/configs/eval.py

@@ -33,7 +33,7 @@ class EvalConfig(EnvServerConfig):
             "group_size", "rollouts_per_example", "num_rollouts", "r"
         ),
     )
-    """Independent env-rollouts per task — the trainer's group size."""
+    """Independent episodes per task — the trainer's group size."""
     shuffle: bool = Field(False, validation_alias=AliasChoices("shuffle", "s"))
     """Shuffle tasks before taking the first `num_tasks`."""
     max_concurrent: int | None = Field(

{verifiers-0.2.2.dev9 → verifiers-0.2.2.dev11}/verifiers/v1/env.py

@@ -1,4 +1,4 @@
-"""Compose a taskset, its agents, and how one task becomes one env-rollout."""
+"""Compose a taskset, its agents, and how one task becomes one episode."""
 
 import asyncio
 import contextlib
@@ -196,7 +196,7 @@ def _as_error(e: Exception) -> Error:
 
 @dataclass
 class RunSlot:
-    """One planned env-rollout, observable while it happens: `traces` collects the
+    """One planned episode, observable while it happens: `traces` collects the
     current attempt (a retry restarts the list), `episode`/`done` land when final."""
 
     task: Task
@@ -218,7 +218,7 @@ ConfigT = TypeVar("ConfigT", bound=EnvConfig)
 
 
 class Env(ABC, Generic[ConfigT]):
-    """A taskset, the agents that play it, and how one task becomes one env-rollout.
+    """A taskset, the agents that play it, and how one task becomes one episode.
 
     Abstract: every run gets a concrete subclass — `SingleAgentEnv` for every plain
     taskset. A multi-agent env declares each role as an `AgentConfig` field on an
@@ -298,10 +298,10 @@ class Env(ABC, Generic[ConfigT]):
 
     @abstractmethod
     async def run(self, task: Task, agents: Agents) -> None:
-        """One env-rollout: how the agents interact on `task`, returning nothing —
+        """One episode: how the agents interact on `task`, returning nothing —
         every finished run joins the episode automatically, stamped with its seat's
         standing. An agent-run failure is data on its trace (this hook decides what
-        it means); an exception raised here is the env-rollout itself failing."""
+        it means); an exception raised here is the episode itself failing."""
 
     async def finalize(self, task: Task, episode: Episode) -> None:
         """Cross-agent judgement — THE programmable judgement surface: plain
@@ -309,11 +309,11 @@ class Env(ABC, Generic[ConfigT]):
         ran on each trace's own task). `episode.traces` is the flat episode in
         completion order, each trace's `agent_name` stamp naming its agent; attach
         signals via `record_reward`/`record_metric`, in program order. A raise
-        fails the env-rollout (the retryable unit) — validate strictly, never
+        fails the episode (the retryable unit) — validate strictly, never
         record a guess."""
 
     def complete(self, episode: Episode) -> bool:
-        """Whether a finished env-rollout is a valid result — what `--resume` keeps
+        """Whether a finished episode is a valid result — what `--resume` keeps
         vs. redoes (default `episode.ok`). An env whose `run()` tolerates a
         failed participant overrides this, else resume re-runs accepted rollouts."""
         return episode.ok
@@ -340,7 +340,7 @@ class Env(ABC, Generic[ConfigT]):
         on_trace: Callable[[Trace], None] | None,
         on_discard: Callable[[Trace], None] | None,
     ) -> Agents:
-        """One env-rollout's `Agents` — fresh value objects riding the live serving
+        """One episode's `Agents` — fresh value objects riding the live serving
         resources (nothing shared across concurrent episodes); `setup()` sees them first."""
 
         def make(name: str, spec: AgentConfig) -> Agent:
@@ -391,7 +391,7 @@ class Env(ABC, Generic[ConfigT]):
         on_discard: Callable[[Trace], None] | None = None,
         gate: asyncio.Semaphore | None = None,
     ) -> Episode:
-        """One env-rollout of `task`, minted as the wire atom: `setup()` then `run()`
+        """One episode of `task`, minted as the wire atom: `setup()` then `run()`
         over fresh agents, then `finalize()`; `gate` bounds
         the agent runs, so internal fan-out counts against `--max-concurrent` too.
         Traces join the episode as runs complete — a hook raising mid-way yields the
@@ -440,7 +440,7 @@ class Env(ABC, Generic[ConfigT]):
         return episode
 
     def slots(self, task: Task, n: int = 1) -> list[RunSlot]:
-        """Plan `n` independent env-rollouts of `task` (`-r n`): nothing couples them."""
+        """Plan `n` independent episodes of `task` (`-r n`): nothing couples them."""
         if n < 1:
             raise ValueError("a task needs at least one rollout (n >= 1)")
         return [RunSlot(task) for _ in range(n)]
@@ -452,7 +452,7 @@ class Env(ABC, Generic[ConfigT]):
         semaphore: asyncio.Semaphore | None = None,
         on_complete: Callable[[Episode], Awaitable[None]] | None = None,
     ) -> Episode:
-        """Run one planned env-rollout to its finished episode, with whole-episode
+        """Run one planned episode to completion, with whole-episode
         retries per `--env.retries`; `semaphore` gates the agent RUNS, not the
         episode; `on_complete` (the runners' persistence hook) fires when final."""
 

{verifiers-0.2.2.dev9 → verifiers-0.2.2.dev11}/verifiers/v1/envs/agentic_judge/env.py

@@ -152,7 +152,7 @@ class AgenticJudgeEnv(vf.Env[AgenticJudgeEnvConfig]):
     async def finalize(self, task: vf.Task, episode: vf.Episode) -> None:
         """Record the scraped verdict on the SOLVER's trace. Strict on scale: an
         off-scale score raises (a judge answering `95` must not clamp to full
-        marks), failing the env-rollout rather than scoring the solver wrong."""
+        marks), failing the episode rather than scoring the solver wrong."""
         by_agent = {t.agent_name: t for t in episode.traces}
         solution, verdict = by_agent["solver"], by_agent["judge"]
         data = verdict.info.get("verdict")

{verifiers-0.2.2.dev9 → verifiers-0.2.2.dev11}/verifiers/v1/envs/best_of_n/env.py

@@ -18,7 +18,7 @@ class BestOfNEnvConfig(vf.EnvConfig):
     # composes with a plain run's `--env.agent.*` flags unchanged.
     agent: vf.AgentConfig = vf.AgentConfig()
     n: int = Field(4, ge=1)
-    """Independent attempts per env-rollout, scored against each other."""
+    """Independent attempts per episode, scored against each other."""
     threshold: float = 1.0
     """A sibling counts as solved when its task reward reaches this (`pass_at_n`)."""
 

{verifiers-0.2.2.dev9 → verifiers-0.2.2.dev11}/verifiers/v1/episode.py

@@ -1,4 +1,4 @@
-"""The episode — one env-rollout's traces plus their shared standing, whole."""
+"""The episode — one run's traces plus their shared standing, whole."""
 
 import uuid
 from typing import Generic
@@ -12,7 +12,7 @@ from verifiers.v1.types import StrictBaseModel
 
 
 class Episode(StrictBaseModel, Generic[DataT, StateT]):
-    """One env-rollout, whole: its identity and standing (`id`, `env`, `errors`)
+    """One run of a task, whole: its identity and standing (`id`, `env`, `errors`)
     next to its flat `traces` — the object `finalize()` receives, the engine
     returns, and the durability envelope: one episode is one `traces.jsonl` line
     and one serve reply, so it persists and arrives whole or not at all — a torn

{verifiers-0.2.2.dev9 → verifiers-0.2.2.dev11}/verifiers/v1/gepa/adapter.py

@@ -54,7 +54,7 @@ class GEPAAdapter:
         capture_traces: bool = False,
     ) -> EvaluationBatch[Episode, Episode]:
         """Run `candidate`'s system prompt on the tasks named by `batch` (`Task.idx` values)
-        and score them — one env-rollout and one score per batch entry, so the batch stays
+        and score them — one episode and one score per batch entry, so the batch stays
         aligned however many traces the env's episode holds. Called synchronously by GEPA on
         the main thread; each batch's rollouts run on the runner's persistent loop via
         `run_until_complete`."""
@@ -122,7 +122,7 @@ class GEPAAdapter:
 
 
 def _episode_score(episode: Episode) -> float:
-    """A candidate's score on one env-rollout: the mean reward of the episode's scored
+    """A candidate's score on one episode: the mean reward of the episode's scored
     traces. Seats that recorded no rewards (a reward-less judge) don't dilute the
     signal; an episode with no scored traces scores 0."""
     scored = [trace.reward for trace in episode.traces if trace.rewards]

{verifiers-0.2.2.dev9 → verifiers-0.2.2.dev11}/verifiers/v1/harnesses/rlm/harness.py

@@ -90,16 +90,16 @@ class RLMHarness(Harness[RLMHarnessConfig]):
         if result.exit_code != 0:
             raise RuntimeError(f"rlm install failed: {result.stderr.strip()[-500:]}")
 
-    def summarize_threshold(self, task_idx: int) -> str:
-        """The `RLM_SUMMARIZE_AT_TOKENS` value: a range draws per-group (seeded by task index, so
-        a task's rollouts share one threshold). Always set — "" when disabled — so the typed field,
-        not a host var the subprocess runtime would inherit, wins."""
+    def summarize_threshold(self, task_idx: int | None) -> str:
+        """The `RLM_SUMMARIZE_AT_TOKENS` value: a range draws per-group (seeded by task index —
+        0 when unset — so a task's rollouts share one threshold). Always set — "" when disabled —
+        so the typed field, not a host var the subprocess runtime would inherit, wins."""
         value = self.config.summarize_at_tokens
         if value is None:
             return ""
         if isinstance(value, tuple):
             lo, hi = value
-            return str(random.Random(task_idx).randint(lo, hi))
+            return str(random.Random(task_idx or 0).randint(lo, hi))
         return str(value)
 
     async def launch(

{verifiers-0.2.2.dev9 → verifiers-0.2.2.dev11}/verifiers/v1/retries.py

@@ -5,7 +5,7 @@ framework adds targeted retries only where no SDK sits underneath (`retrying()`)
 Two opt-in whole-run retry atoms sit above that: `Agent.run` reruns ITS OWN rollout
 while the trace ends with a retryable error (`--env.<agent>.retries` — a flaky
 grader retries without re-burning the solver), and `run_episode_with_retry` reruns
-the entire env-rollout (`--env.retries`) — the coarse fallback for faults no agent
+the entire episode (`--env.retries`) — the coarse fallback for faults no agent
 owns: the env's own hooks, cross-agent state. Both match by exception type name;
 both off by default.
 """
@@ -105,7 +105,7 @@ def trace_should_retry(trace, retry: RetryConfig) -> bool:
 
 
 def episode_should_retry(episode: Episode, retry: RetryConfig) -> bool:
-    """Whether a finished env-rollout should be retried: any LIVE captured error —
+    """Whether a finished episode should be retried: any LIVE captured error —
     episode-level, or on a trace that ended not-`ok` — is retryable. An `ok` trace's
     errors are history its own per-agent retry already recovered from, never
     grounds to re-run the episode. All of a failed trace's captures count, not just
@@ -121,7 +121,7 @@ async def run_episode_with_retry(
     run: Callable[[], Awaitable[Episode]],
     retry: RetryConfig,
 ) -> Episode:
-    """Run one env-rollout (`run` must mint a fresh episode per call), retrying while
+    """Run one episode (each attempt mints a fresh one), retrying while
     it ends with a retryable error. When the final attempt fails too, the earlier
     attempts' errors are prepended so the episode shows the full history; a final
     good attempt returns clean."""
@@ -136,7 +136,7 @@ async def run_episode_with_retry(
             history.extend(trace.errors)
         delay = backoff(attempt)
         logger.warning(
-            "retrying env-rollout %s (retry %d/%d) in %.1fs after error: %s",
+            "retrying episode %s (retry %d/%d) in %.1fs after error: %s",
             final.id,
             attempt + 1,
             retry.max_retries,

{verifiers-0.2.2.dev9 → verifiers-0.2.2.dev11}/verifiers/v1/task.py

@@ -125,7 +125,9 @@ class TaskData(StrictBaseModel):
 
     model_config = ConfigDict(frozen=True)
 
-    idx: int
+    idx: int | None = None
+    """Dataset row index — set by tasksets (selection, resume, and grouping key
+    on the eval path); `None` for ad-hoc tasks minted in scripts."""
     name: str | None = None
     description: str | None = None
     prompt: str | Messages | None = None