verifiers 0.2.2.dev65__tar.gz → 0.2.2.dev67__tar.gz

{verifiers-0.2.2.dev65 → verifiers-0.2.2.dev67}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.2.dev65
+Version: 0.2.2.dev67
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.2.2.dev65 → verifiers-0.2.2.dev67}/tests/v1/test_e2e.py

@@ -479,10 +479,10 @@ async def test_env_id_user_sim(run_v1, tmp_path):
     assert user.agent.trainable is False
     assert user.num_turns >= 1  # the modeled user actually spoke
     assert assistant.metrics["user_turns"] >= 1
-    # `mask_prompt`: the scenario is hidden from the assistant's harness (the run's
-    # visible data) while the task's own rewards still scored the real row. The
-    # masked view is what persists (provenance is the row's idx); both sides land
-    # as ONE durable episode.
+    # The env nulls the assistant task's prompt: the scenario is hidden from the
+    # assistant's harness while the task's rewards score off non-prompt fields
+    # (`answer`). The nulled row is what persists (provenance is the row's idx);
+    # both sides land as ONE durable episode.
     assert assistant.task.data.prompt is None
     assert "echoed" in assistant.rewards
     from verifiers.v1.cli.output import read_episodes

{verifiers-0.2.2.dev65 → verifiers-0.2.2.dev67}/verifiers/v1/agent.py

@@ -21,10 +21,11 @@ from verifiers.v1.clients import (
     resolve_client,
 )
 from verifiers.v1.configs.agent import AgentConfig, TimeoutConfig
+from verifiers.v1.dialects import parse_message
 from verifiers.v1.harness import Harness
 from verifiers.v1.interception import Interception, InterceptionServer
 from verifiers.v1.mcp import SharedToolServer
-from verifiers.v1.rollout import RolloutRun, RolloutTimeouts, _as_messages
+from verifiers.v1.rollout import Rollout, RolloutTimeouts
 from verifiers.v1.runtimes import (
     NetworkPolicyConfig,
     Runtime,
@@ -150,9 +151,7 @@ class Interaction:
     rewards after close. Leaving the `interaction()` context closes the exchange
     as `user_closed` and finishes the rollout — hooks and scoring included."""
 
-    def __init__(
-        self, run: "RolloutRun", gate: asyncio.Semaphore | None = None
-    ) -> None:
+    def __init__(self, run: "Rollout", gate: asyncio.Semaphore | None = None) -> None:
         self._run = run
         self._gate = gate
         self._over = False  # a terminated segment was already delivered
@@ -188,14 +187,16 @@ class Interaction:
         if message is not None and prompted:
             raise ValueError(
                 "the task's prompt opens this exchange: take its first reply with "
-                "a bare turn() before answering (or mask the prompt with "
-                "interaction(mask_prompt=True) to open the conversation yourself)"
+                "a bare turn() before answering (or hand the interaction a task "
+                "with `prompt=None` to open the conversation yourself)"
             )
         messages: Messages | None = None
         if isinstance(message, str):
             messages = [UserMessage(content=message)]
         elif message is not None:
-            messages = _as_messages(message)
+            # A turn's messages may arrive typed or as wire dicts (env code naturally
+            # writes `{"role": "user", ...}`); the trace speaks typed, so normalize.
+            messages = [parse_message(m) if isinstance(m, dict) else m for m in message]
         self._started = True
         turns_before = self.trace.num_turns
         nodes_before = len(self.trace.nodes)
@@ -411,7 +412,7 @@ class Agent:
         on_trace: Callable[[Trace], None] | None,
     ) -> Trace:
         params = self._rollout_params(task, runtime, dict(shared_tools or {}))
-        run = RolloutRun(task=task, on_trace=on_trace, **params)
+        run = Rollout(task=task, on_trace=on_trace, **params)
         try:
             if await run.open():
                 await run.step()
@@ -434,7 +435,6 @@ class Agent:
         *,
         runtime: Runtime | None = None,
         tools: Mapping[str, SharedToolServer] | None = None,
-        mask_prompt: bool = False,
         on_trace: Callable[[Trace], None] | None = None,
     ) -> AsyncIterator[Interaction]:
         """Interact with this agent turn-by-turn: a full rollout of `task` where
@@ -446,11 +446,10 @@ class Agent:
 
         The task's shape says who speaks first: a prompt-less task is opened by
         the first `turn(message)`; a prompted task speaks first — take its opening
-        reply with a bare `turn()` before answering. `mask_prompt` says a prompted
-        task's prompt belongs to the USER side (a scenario the caller pursues, not
-        the assistant's seed): the wire hides it — the caller opens — while the
-        task object keeps the full row, so its hooks, rewards, and judges score
-        the real question (the user-sim env's contract).
+        reply with a bare `turn()` before answering. A prompt that belongs to the
+        USER side (a scenario the caller pursues, not the assistant's seed) is the
+        caller's to hide: hand the interaction a task whose `data.prompt` is None
+        and keep the scenario on a scoring-side field (the user-sim env's contract).
 
         `runtime` and `tools` borrow live resources from their owners, just as
         they do for `run()`; an env supplies its taskset's shared tools
@@ -466,17 +465,9 @@ class Agent:
         if self._closed:
             raise RuntimeError("Agent is closed; create a new agent")
         self._check_resume_support()
-        if mask_prompt and task.data.prompt is None:
-            raise ValueError(
-                "mask_prompt hides a prompt the task doesn't have; a prompt-less "
-                "task is already opened by the first turn()"
-            )
         params = self._rollout_params(task, runtime, dict(tools or {}))
-        run = RolloutRun(
+        run = Rollout(
             task=task,
-            wire_data=(
-                task.data.model_copy(update={"prompt": None}) if mask_prompt else None
-            ),
             has_user=True,
             on_trace=on_trace,
             **params,
@@ -665,7 +656,6 @@ class _EpisodeAgent(Agent):
         *,
         runtime: Runtime | None = None,
         tools: Mapping[str, SharedToolServer] | None = None,
-        mask_prompt: bool = False,
         on_trace: Callable[[Trace], None] | None = None,
     ) -> AsyncIterator[Interaction]:
         """The agent's `interaction`, with every trace stamped with its standing
@@ -687,7 +677,6 @@ class _EpisodeAgent(Agent):
                 task,
                 runtime=runtime,
                 tools=tools if tools is not None else self._shared_for(task),
-                mask_prompt=mask_prompt,
                 on_trace=self._watch(remember),
             ) as interaction:
                 yield interaction

{verifiers-0.2.2.dev65 → verifiers-0.2.2.dev67}/verifiers/v1/configs/agent.py

@@ -11,44 +11,43 @@ from verifiers.v1.types import SamplingConfig
 
 
 class TimeoutConfig(BaseConfig):
-    """Per-agent wall-clock timeouts per rollout stage, in seconds (None = no
-    limit); each stage falls back to the task's own `TaskTimeout` when unset. An
-    interaction's rollout budget is cumulative across its active harness segments
-    and pauses while the caller computes the next user turn."""
+    """Timeout (in seconds) for different phases of an agent's run."""
 
     setup: float | None = None  # one shared budget: task setup + provisioning
+    """Timeout (in seconds) for the task + harness setup hooks."""
     rollout: float | None = None
+    """Timeout (in seconds) for the agent's solve attempt."""
     finalize: float | None = None
+    """Timeout (in seconds) for the task + harness finalize hooks."""
     scoring: float | None = None
+    """Timeout (in seconds) for the task + harness metrics + scoring hooks."""
 
 
 class AgentConfig(BaseConfig):
-    """One env agent: who plays it, and its per-run caps. It pins only what
-    makes it a different actor; everything unpinned falls back — the model context
-    to the run's own, the harness to the taskset's default."""
-
     harness: SerializeAsAny[HarnessConfig] | None = None
     """The agent's program (None = the taskset's default harness)."""
     runtime: RuntimeConfig = SubprocessConfig()
     """Runtime for the harness program — the policy each run provisions its box
     from; tool servers choose their placement separately."""
+
     model: str | None = None
     """Model id (None = the run's model, i.e. the policy under evaluation/training)."""
     client: ClientConfig | None = None
     """Endpoint override (None = the run's client)."""
     sampling: SamplingConfig | None = None
     """Sampling override (None = the run's sampling)."""
-    timeout: TimeoutConfig = TimeoutConfig()
-    retries: RetryConfig = RetryConfig()
-    """Whole-run retries: rerun this agent's rollout while its trace ends with a
-    retryable error (never into a borrowed box)."""
+
     max_turns: int | None = None
-    """Max model turns per run (None = no limit). Framework-enforced (the
-    interception server refuses turns past it), so it applies to any harness."""
+    """Max model turns per run (None = no limit)."""
     max_input_tokens: int | None = None
+    """Max input tokens per run (None = no limit)."""
     max_output_tokens: int | None = None
+    """Max output tokens per run (None = no limit)."""
     max_total_tokens: int | None = None
-    """Token caps per run (None = no limit); framework-enforced between turns."""
+    """Max total tokens per run (None = no limit)."""
+
+    timeout: TimeoutConfig = TimeoutConfig()
+    retries: RetryConfig = RetryConfig()
 
     @model_validator(mode="before")
     @classmethod

{verifiers-0.2.2.dev65 → verifiers-0.2.2.dev67}/verifiers/v1/envs/user_sim/env.py

@@ -3,10 +3,11 @@
 The generic two-sided conversation (`--env.id user-sim` over any taskset) — the
 substrate a tau2-style benchmark builds on. The taskset's row is read as the USER's
 side of the world: its prompt text becomes the scenario in the user's system prompt
-(`--env.persona`), and the assistant plays the same task with a masked prompt — it
-is hidden from its harness, so it learns the user's goal only through
-conversation (its own instructions stay in `system_prompt`), while the task's
-rewards and judges still score the real row. The user agent rides the tool-less
+(`--env.persona`), and the assistant plays the same task with its prompt nulled — the
+scenario is the user's knowledge, so the assistant learns the goal only through
+conversation (its own instructions stay in `system_prompt`). The task's rewards and
+judges score that nulled row, so they must read scoring-side fields (`answer`, a
+judge's `question_field`), never the prompt. The user agent rides the tool-less
 `null` harness by default (untrainable — `setup()`), opens the conversation, and
 ends it with the done marker; the assistant's trace is then judged by the task's
 own rewards, exactly as in any eval.
@@ -64,14 +65,16 @@ class UserSimEnv(vf.Env[UserSimEnvConfig]):
             )
         )
         # Two interactions, relayed: the user is just another agent, and the env is
-        # the control flow between them. The assistant plays the SAME task with a
-        # masked prompt: the scenario is the user's knowledge, so the wire seeds
-        # nothing and the user opens — while the task's hooks, rewards, and plugged
-        # judges still score the real row (they read the task object, not the
-        # run's masked view).
+        # the control flow between them. The assistant plays the SAME task with its
+        # prompt nulled: the scenario is the user's knowledge, so the wire seeds
+        # nothing and the user opens. Scoring runs on the nulled row — a user-sim
+        # taskset keeps its scoring on non-prompt fields.
+        assistant_task = type(task)(
+            task.data.model_copy(update={"prompt": None}), task.config
+        )
         async with (
             agents.user.interaction(user_task) as sim,
-            agents.assistant.interaction(task, mask_prompt=True) as assistant,
+            agents.assistant.interaction(assistant_task) as assistant,
         ):
             # The tau convention: the assistant "answers the phone", the user
             # states the goal. The greeting exists only on the user's side. A

{verifiers-0.2.2.dev65 → verifiers-0.2.2.dev67}/verifiers/v1/errors.py

@@ -13,7 +13,7 @@ Four mechanisms, each in one place:
 3. Surfacing (`session.RolloutSession.error`): a model or tool call fails behind the harness
    subprocess and comes back as HTTP, so the interception server stashes the real error there and
    the rollout re-raises it once the harness returns — not a secondary `HarnessError`.
-4. Capture (`RolloutRun`, mirrored by the env-server): the one place that records a failure (typed
+4. Capture (`Rollout`, mirrored by the env-server): the one place that records a failure (typed
    or not) onto the trace and never lets it cancel sibling rollouts. A bad rollout is data, not a
    crash.
 

{verifiers-0.2.2.dev65 → verifiers-0.2.2.dev67}/verifiers/v1/interception/__init__.py

@@ -1,4 +1,5 @@
-from collections.abc import Iterable
+from collections.abc import AsyncIterator, Iterable
+from contextlib import asynccontextmanager
 from typing import TYPE_CHECKING, Annotated
 
 from pydantic import Field
@@ -15,7 +16,8 @@ from verifiers.v1.interception.server import (
     InterceptionServer,
     InterceptionServerConfig,
 )
-from verifiers.v1.runtimes import runtime_is_local
+from verifiers.v1.runtimes import Runtime, runtime_is_local
+from verifiers.v1.session import RolloutSession
 
 if TYPE_CHECKING:
     from verifiers.v1.mcp import SharedToolServer
@@ -69,6 +71,36 @@ def make_interception(
     return ElasticInterceptionPool(config, requires_tunnel, state_service_secrets)
 
 
+@asynccontextmanager
+async def serve_interception(
+    interception: Interception | None,
+    runtime: Runtime,
+    session: RolloutSession,
+    servers: list,
+    shared_tools: "dict[str, SharedToolServer]",
+) -> AsyncIterator[Slot]:
+    """A slot on the shared interception when one was injected (its owner keeps the
+    lifecycle), else on a per-rollout `InterceptionServer` owned — brought up and torn
+    down — by the caller's context."""
+    if interception is not None:
+        async with interception.acquire(session) as slot:
+            yield slot
+        return
+    tunneled = requires_tunnel(
+        runtime.is_local,
+        [server.config for server in servers],
+        shared_tools.values(),
+    )
+    server = InterceptionServer(
+        requires_tunnel=tunneled,
+        state_service_secrets=tuple(
+            tool.state_secret for tool in shared_tools.values() if tool.state_secret
+        ),
+    )
+    async with server, server.acquire(session) as slot:
+        yield slot
+
+
 __all__ = [
     "BaseInterceptionConfig",
     "ElasticInterceptionPool",
@@ -82,4 +114,5 @@ __all__ = [
     "StaticInterceptionPoolConfig",
     "make_interception",
     "requires_tunnel",
+    "serve_interception",
 ]

{verifiers-0.2.2.dev65 → verifiers-0.2.2.dev67}/verifiers/v1/rollout.py

@@ -1,33 +1,15 @@
-"""A rollout: one trajectory — drive a harness segment by segment and score its trace.
-
-A rollout's exchange is a sequence of SEGMENTS: the harness program runs until it
-yields (= exits), the run's user answers its final message, and the next segment
-resumes the exchange with that answer (`Harness.resume` — a relaunch on the accreted
-conversation by default, a native continuation for harnesses with their own session
-state). The user loop lives between segments, at the exchange's natural turn
-granularity — never inside the model boundary, so a harness's own tool loop can
-never race or amputate it.
-
-`RolloutRun` is the engine, a staged lifecycle: `open()` boots the world, each
-`step()` runs one segment, `close()` finalizes, scores, and tears the world down —
-each stage under its own timeout. `Agent` is its only driver: `Agent.run` is the
-one-call single-segment form, `Agent.interaction` holds the run open and lets the
-caller supply each user turn, one `turn()` per segment — who answers the program
-(an env's control flow, a simulator agent, a game engine, a human) is the caller's
-business, never this module's.
-"""
+"""Lifecycle of one agent rollout."""
 
 import asyncio
 import contextlib
 import logging
 import time
-from collections.abc import AsyncIterator, Callable
-from contextlib import AsyncExitStack, asynccontextmanager
+from collections.abc import Callable
+from contextlib import AsyncExitStack
 from dataclasses import dataclass
 
 from verifiers.v1.clients import ModelContext
 from verifiers.v1.configs.agent import AgentConfig
-from verifiers.v1.dialects import parse_message
 from verifiers.v1.errors import (
     HarnessError,
     RolloutError,
@@ -36,12 +18,7 @@ from verifiers.v1.errors import (
     boundary,
 )
 from verifiers.v1.harness import Harness
-from verifiers.v1.interception import (
-    Interception,
-    InterceptionServer,
-    Slot,
-    requires_tunnel,
-)
+from verifiers.v1.interception import Interception, serve_interception
 from verifiers.v1.mcp import SharedToolServer, serve_tools
 from verifiers.v1.runtimes import (
     Runtime,
@@ -50,7 +27,7 @@ from verifiers.v1.runtimes import (
 )
 from verifiers.v1.session import RolloutLimits, RolloutSession
 from verifiers.v1.state import state_cls
-from verifiers.v1.task import Task, TaskData
+from verifiers.v1.task import Task
 from verifiers.v1.trace import AgentInfo, Trace, TraceTask
 from verifiers.v1.types import Messages
 from verifiers.v1.utils.decorators import discover_decorated, invoke
@@ -60,69 +37,20 @@ logger = logging.getLogger(__name__)
 
 @dataclass(frozen=True)
 class RolloutTimeouts:
-    """Per-stage rollout timeouts in seconds (None = unlimited), each bounding one
-    lifecycle stage: `setup` covers task + harness setup in `open()`, `agent` is the
-    cumulative budget the run's segments draw down across `step()`s, `finalize` and
-    `scoring` bound their `close()` stages."""
+    """Per-stage rollout timeouts, each bounding one rollout stage."""
 
     setup: float | None = None
+    """Timeout (in seconds) for the task + harness setup hooks."""
     agent: float | None = None
+    """Timeout (in seconds) for the agent's solve attempt."""
     finalize: float | None = None
+    """Timeout (in seconds) for the task + harness finalize hooks."""
     scoring: float | None = None
+    """Timeout (in seconds) for the task + harness metrics + scoring hooks."""
 
 
-def _as_messages(raw: Messages) -> Messages:
-    """A turn's messages may arrive typed or as wire dicts (env code naturally
-    writes `{"role": "user", ...}`); the trace speaks typed, so normalize here."""
-    return [parse_message(m) if isinstance(m, dict) else m for m in raw]
-
-
-@asynccontextmanager
-async def _serve_interception(
-    interception: Interception | None,
-    runtime: Runtime,
-    session: RolloutSession,
-    servers: list,
-    shared_tools: dict[str, SharedToolServer],
-) -> AsyncIterator[Slot]:
-    """A slot on the shared interception when one was injected (its owner keeps the
-    lifecycle), else on a per-rollout `InterceptionServer` owned — brought up and torn
-    down — by this rollout."""
-    if interception is not None:
-        async with interception.acquire(session) as slot:
-            yield slot
-        return
-    tunneled = requires_tunnel(
-        runtime.is_local,
-        [server.config for server in servers],
-        shared_tools.values(),
-    )
-    server = InterceptionServer(
-        requires_tunnel=tunneled,
-        state_service_secrets=tuple(
-            tool.state_secret for tool in shared_tools.values() if tool.state_secret
-        ),
-    )
-    async with server, server.acquire(session) as slot:
-        yield slot
-
-
-class RolloutRun:
-    """One rollout held open segment by segment.
-
-    `open()` boots the world (runtime, setup, interception, tool servers); each
-    `step()` runs ONE harness segment — a program run to its exit — resuming the
-    exchange with the user turn(s) it's given; `close()` finalizes, scores, and
-    tears the world down, returning the finished trace. Expected `RolloutError`s
-    are captured onto the trace (a bad rollout is data, not a crash): `open` and
-    `step` report continuability as a bool, and `close` always returns the trace.
-
-    `wire_data` is the run's recorded view of the task — what `trace.task.data`
-    says the harness saw (`Agent.interaction(mask_prompt=True)` masks the prompt here
-    while the `task` object keeps the full row for its hooks and judges).
-    `runtime` is a live box to run in instead of provisioning one; a borrowed
-    runtime is neither started nor stopped here. `on_trace` observes the run's
-    trace the moment it's minted, before any I/O."""
+class Rollout:
+    """Manages one rollout's lifecycle (open, step, close)."""
 
     def __init__(
         self,
@@ -132,10 +60,9 @@ class RolloutRun:
         harness: Harness,
         ctx: ModelContext,
         runtime_config: RuntimeConfig,
-        wire_data: TaskData | None = None,
         has_user: bool = False,
-        timeouts: RolloutTimeouts | None = None,
-        limits: RolloutLimits | None = None,
+        timeouts: RolloutTimeouts,
+        limits: RolloutLimits,
         shared_tools: dict[str, SharedToolServer] | None = None,
         interception: Interception | None = None,
         runtime: Runtime | None = None,
@@ -146,7 +73,7 @@ class RolloutRun:
         self.ctx = ctx
         self.runtime_config = runtime_config
         self._has_user = has_user
-        self._timeouts = timeouts or RolloutTimeouts()
+        self._timeouts = timeouts
         self._agent_time_remaining = self._timeouts.agent
         self._shared_tools = shared_tools or {}
         self._interception = interception
@@ -155,7 +82,7 @@ class RolloutRun:
         self.trace: Trace = Trace(
             task=TraceTask(
                 type=type(task).__name__,
-                data=task.data if wire_data is None else wire_data,
+                data=task.data,
             ),
             state=state_cls(type(task))(),
             # The seat's resolved config, role overrides included — the agent
@@ -165,7 +92,7 @@ class RolloutRun:
         if on_trace is not None:
             on_trace(self.trace)
         self._session = RolloutSession(
-            ctx, self.trace, discover_decorated(task, "stop"), limits or RolloutLimits()
+            ctx, self.trace, discover_decorated(task, "stop"), limits
         )
         self._stack = AsyncExitStack()
         self._failed = False
@@ -281,7 +208,7 @@ class RolloutRun:
                 model_secret,
                 state_secret,
             ) = await self._stack.enter_async_context(
-                _serve_interception(
+                serve_interception(
                     self._interception,
                     runtime,
                     self._session,

{verifiers-0.2.2.dev65 → verifiers-0.2.2.dev67}/verifiers/v1/utils/retries.py

@@ -117,7 +117,7 @@ async def run_episode_with_retry(
         final = await run()
         if attempt == retry.max_retries or not episode_should_retry(final, retry):
             break
-        cause = final.error or next(
+        cause = final.last_error or next(
             (t.last_error for t in final.traces if t.last_error), None
         )
         history.extend(final.errors)