verifiers 0.2.1.dev21__tar.gz → 0.2.1.dev23__tar.gz

{verifiers-0.2.1.dev21 → verifiers-0.2.1.dev23}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.1.dev21
+Version: 0.2.1.dev23
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.2.1.dev21 → verifiers-0.2.1.dev23}/verifiers/v1/cli/eval/runner.py

@@ -70,7 +70,7 @@ async def run_eval(env: Environment, config: EvalConfig) -> list[Trace]:
         await append_trace(out, trace, write_lock)
 
     # Shared tool servers (if any) come up once here and their URLs flow into every rollout
-    # (non-shared ones start per rollout inside the episodes); the interception pool comes up
+    # (non-shared ones start per rollout inside the episodes); the interception comes up
     # here too, so concurrent rollouts share its servers + tunnels rather than one each. Build
     # episodes inside `serving` so each rollout is wired to those resources at construction.
     async with env.serving():

{verifiers-0.2.1.dev21 → verifiers-0.2.1.dev23}/verifiers/v1/cli/gepa.py

@@ -60,7 +60,7 @@ def main(argv: list[str] | None = None) -> None:
         return
 
     # First Ctrl-C / SIGTERM warns and raises KeyboardInterrupt so a killed/timed-out run still
-    # runs the runner's `serving()` teardown (interception pool / tool-server runtimes); further
+    # runs the runner's `serving()` teardown (interception / tool-server runtimes); further
     # signals during that cleanup are swallowed so an impatient second Ctrl-C can't orphan them.
     install_interrupt()
 

{verifiers-0.2.1.dev21 → verifiers-0.2.1.dev23}/verifiers/v1/env.py

@@ -12,7 +12,14 @@ from verifiers.v1.clients import ModelContext
 from verifiers.v1.decorators import discover_decorated
 from verifiers.v1.episode import Episode
 from verifiers.v1.types import ID
-from verifiers.v1.interception import InterceptionPool, RolloutLimits
+from verifiers.v1.interception import (
+    ElasticInterceptionPoolConfig,
+    Interception,
+    InterceptionConfig,
+    make_interception,
+    requires_tunnel,
+)
+from verifiers.v1.session import RolloutLimits
 from verifiers.v1.retries import RetryConfig
 from verifiers.v1.rollout import Rollout
 from verifiers.v1.runtimes import (
@@ -20,7 +27,7 @@ from verifiers.v1.runtimes import (
     SubprocessConfig,
     runtime_is_local,
 )
-from verifiers.v1.task import Task
+from verifiers.v1.task import Task, resolve_server_config
 from verifiers.v1.taskset import Taskset, TasksetConfig
 from verifiers.v1.utils.generic import generic_type
 from verifiers.v1.mcp import SharedToolServer, serve_shared
@@ -100,10 +107,11 @@ class EnvConfig(BaseConfig):
     max_total_tokens: int | None = None
     """Max total (prompt + completion) tokens per rollout (None = no limit). Caps the
     trace's `num_total_tokens`; framework-enforced between turns."""
-    multiplex: int = Field(32, ge=1)
-    """Rollouts that share one interception server (and, behind a remote runtime, one
-    tunnel). N concurrent rollouts use ~N/multiplex servers + tunnels instead of one each —
-    key past the per-token tunnel cap. 1 = a server (+ tunnel) per rollout."""
+    interception: InterceptionConfig = ElasticInterceptionPoolConfig()
+    """The interception shape (see `verifiers.v1.interception`): `elastic` (the
+    default — servers grown on demand, `multiplex` rollouts each), `server` (one server,
+    with a tunnel choice incl. a bring-your-own endpoint), or `static` (a fixed list of
+    such servers)."""
     # --- legacy (v0) backwards-compat -----------------------------------------
     id: ID | None = None
     """Classic (v0) env id (`name`, `org/name`, or `org/name@version` — installed from the
@@ -274,9 +282,9 @@ class Environment:
         )
         self._warned_resources: set[tuple[str, str]] = set()
         self._shared_tools: dict[str, SharedToolServer] = {}
-        self._interception: InterceptionPool | None = None
+        self._interception: Interception | None = None
         """Eval-level serving resources, live only inside `serving()`: shared tool servers
-        ({name: SharedToolServer}) and the interception pool. `episode()` injects them into every rollout
+        ({name: SharedToolServer}) and the interception. `episode()` injects them into every rollout
         so neither runner has to thread them through `Episode.run`/`Rollout.run`."""
 
     def runtime_for(self, task: Task) -> RuntimeConfig:
@@ -358,24 +366,44 @@ class Environment:
     @contextlib.asynccontextmanager
     async def serving(self):
         """Hold the env-level serving resources for the duration of an eval: the shared tool
-        servers (built once, see `shared_tools`) and the interception pool. Stash them so
+        servers (built once, see `shared_tools`) and the interception. Stash them so
         every `episode()` built inside this context injects them into its rollouts — that's
         what keeps both eval runners (in-process and env-server) on one serving path. Build
         episodes inside this context; the resources are torn down on exit."""
-        async with (
-            self.shared_tools() as shared,
-            self.interception_pool() as interception,
-        ):
-            self._shared_tools = shared
-            self._interception = interception
-            try:
-                yield
-            finally:
-                self._shared_tools = {}
-                self._interception = None
-
-    def interception_pool(self) -> InterceptionPool:
-        return InterceptionPool(self.harness.config.runtime, self.config.multiplex)
+        async with self.shared_tools() as shared:
+            interception = make_interception(
+                self.config.interception, requires_tunnel=self._requires_tunnel(shared)
+            )
+            async with interception:
+                self._shared_tools = shared
+                self._interception = interception
+                try:
+                    yield
+                finally:
+                    self._shared_tools = {}
+                    self._interception = None
+
+    def _requires_tunnel(self, shared: dict[str, SharedToolServer]) -> bool:
+        """`requires_tunnel` over the consumers known before any rollout: the harness
+        runtime (by config), the live `shared` servers, and the task class's tool/user
+        servers — read class-level (like `validate_pairing`) with their configs resolved
+        the way `Task.server_config` resolves them. A task that *overrides* that pairing
+        isn't statically knowable, so it conservatively counts as remote (the tunnel then
+        reaches everything; a wrongly-assumed localhost would reach nothing remote)."""
+        task_cls = generic_type(type(self.taskset), Task, origin=Taskset) or Task
+        server_classes = [*task_cls.tools, *([task_cls.user] if task_cls.user else [])]
+        if server_classes and task_cls.server_config is not Task.server_config:
+            return True
+        sole = len({*task_cls.tools} | ({task_cls.user} - {None})) == 1
+        configs = [
+            resolve_server_config(
+                task_cls.__name__, self.taskset.config.task, server_cls, sole=sole
+            )
+            for server_cls in server_classes
+        ]
+        return requires_tunnel(
+            runtime_is_local(self.harness.config.runtime), configs, shared.values()
+        )
 
     @contextlib.asynccontextmanager
     async def shared_tools(self):

{verifiers-0.2.1.dev21 → verifiers-0.2.1.dev23}/verifiers/v1/errors.py

@@ -10,7 +10,7 @@ Four mechanisms, each in one place:
    raises plain Python errors — it never constructs a `vf` error type; `boundary` classifies them.
    Infra that fails raises its type at the source (`runtimes` → `SandboxError`, `clients` →
    `ProviderError`, tunnels → `TunnelError`); an already-typed `RolloutError` passes through unchanged.
-3. Surfacing (`interception.RolloutSession.error`): a model/tool/user call fails behind the harness
+3. Surfacing (`session.RolloutSession.error`): a model/tool/user call fails behind the harness
    subprocess and comes back as HTTP, so the interception server stashes the real error there and
    the rollout re-raises it once the harness returns — not a secondary `HarnessError`.
 4. Capture (`Rollout.run`, mirrored by the env-server): the one place that records a failure (typed

verifiers-0.2.1.dev23/verifiers/v1/interception/__init__.py

@@ -0,0 +1,94 @@
+"""The interception layer.
+
+`server` catches a harness's model calls and proxies them to the client, multiplexing many
+rollouts on one server keyed by their secret — and is itself the single-server
+`Interception`. `pool` composes such servers, statically (a fixed set, e.g. bring-your-own
+endpoints) or elastically (grown on demand). `tunnel` makes a server reachable from remote
+consumers; `requires_tunnel` decides whether that's needed. `make_interception` picks the
+shape from the `InterceptionConfig` type.
+"""
+
+from collections.abc import Iterable
+from typing import TYPE_CHECKING, Annotated
+
+from pydantic import Field
+from pydantic_config import BaseConfig
+
+from verifiers.v1.interception.base import BaseInterceptionConfig, Interception, Slot
+from verifiers.v1.interception.pool import (
+    ElasticInterceptionPool,
+    ElasticInterceptionPoolConfig,
+    StaticInterceptionPool,
+    StaticInterceptionPoolConfig,
+)
+from verifiers.v1.interception.server import (
+    InterceptionServer,
+    InterceptionServerConfig,
+)
+from verifiers.v1.runtimes import runtime_is_local
+
+if TYPE_CHECKING:
+    from verifiers.v1.mcp import SharedToolServer
+
+# Discriminated on `type` so the CLI selects with `--interception.type server|static|elastic`.
+InterceptionConfig = Annotated[
+    InterceptionServerConfig
+    | StaticInterceptionPoolConfig
+    | ElasticInterceptionPoolConfig,
+    Field(discriminator="type"),
+]
+
+
+def requires_tunnel(
+    harness_is_local: bool,
+    server_configs: Iterable[BaseConfig] = (),
+    shared: "Iterable[SharedToolServer]" = (),
+) -> bool:
+    """Whether the interception must be exposed via a tunnel — some consumer is off the
+    host network: the harness itself, a live `shared` server in a remote runtime, or a
+    tool/user server config placing one there (each reaches the `/state` channel from
+    its own runtime). Skipped as non-consumers: a `colocated` server (shares the
+    harness's runtime, covered by `harness_is_local`), a config-`url` server (external —
+    it connects out), and an `external` shared server (outside the state machinery
+    entirely). False means every consumer reaches the server at localhost."""
+    if not harness_is_local:
+        return True
+    if any(not s.external and not s.local for s in shared):
+        return True
+    for config in server_configs:
+        if getattr(config, "url", None) or config.colocated:
+            continue
+        if not runtime_is_local(config.runtime):
+            return True
+    return False
+
+
+def make_interception(
+    config: InterceptionConfig, *, requires_tunnel: bool
+) -> Interception:
+    """The interception for a config, picked by type (the host-side counterpart to
+    `make_runtime`). With `requires_tunnel` (some consumer is off the host network — see
+    the `requires_tunnel` util) each server is exposed via its configured tunnel; without
+    it they get none and are reached at localhost. The caller computes it (see
+    `Environment._requires_tunnel`)."""
+    if isinstance(config, InterceptionServerConfig):
+        return InterceptionServer(config, requires_tunnel)
+    if isinstance(config, StaticInterceptionPoolConfig):
+        return StaticInterceptionPool(config, requires_tunnel)
+    return ElasticInterceptionPool(config, requires_tunnel)
+
+
+__all__ = [
+    "Interception",
+    "Slot",
+    "make_interception",
+    "requires_tunnel",
+    "InterceptionServer",
+    "StaticInterceptionPool",
+    "ElasticInterceptionPool",
+    "BaseInterceptionConfig",
+    "InterceptionConfig",
+    "InterceptionServerConfig",
+    "StaticInterceptionPoolConfig",
+    "ElasticInterceptionPoolConfig",
+]

verifiers-0.2.1.dev23/verifiers/v1/interception/base.py

@@ -0,0 +1,68 @@
+"""The interception contract: hand each rollout a slot on a host interception server.
+
+Three shapes, picked by `InterceptionConfig` type (see `make_interception`): a single
+`InterceptionServer`, a fixed `StaticInterceptionPool`, or an on-demand
+`ElasticInterceptionPool`. From the outside they behave the same: start/stop (or the async
+context manager wrapping them) bound the lifecycle — one eval, one topology run, or one
+agent `.run` — and each rollout `acquire`s a slot and frees it. An `Interception` can be
+shared: whoever entered it owns the lifecycle; borrowers only `acquire`.
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from contextlib import AbstractAsyncContextManager, AsyncExitStack
+from typing import TYPE_CHECKING, Self
+
+from pydantic_config import BaseConfig
+
+if TYPE_CHECKING:
+    from verifiers.v1.session import RolloutSession
+
+
+class BaseInterceptionConfig(BaseConfig):
+    """Base for the interception types — the discriminated union's common type. Per-type
+    fields live on the subclasses (server's `tunnel`, static's `servers`, elastic's
+    `multiplex`)."""
+
+
+# (base_url, secret): the interception server's reachable base URL for this rollout, and the
+# bearer the harness/tool/user servers authenticate with. The harness reaches the model at
+# `{base_url}/v1`; tool/user servers reach this rollout's shared state at `{base_url}/state`
+# + `/task`. `base_url` is universally reachable — the interception is exposed (tunnel)
+# whenever any consumer is remote.
+Slot = tuple[str, str]
+
+
+class Interception(ABC):
+    """How rollouts reach the host interception server. `start` brings the servers up (or
+    arms lazy growth), `stop` tears every server (+ its tunnel) down via `stack` — LIFO,
+    even if one teardown fails; `async with` wraps the two. Each rollout `acquire`s a slot
+    and frees it on exit."""
+
+    def __init__(self) -> None:
+        self.stack = AsyncExitStack()
+
+    @abstractmethod
+    async def start(self) -> None:
+        """Bring the interception up; resources land on `stack` so `stop` frees them."""
+
+    async def stop(self) -> None:
+        await self.stack.aclose()
+
+    async def __aenter__(self) -> Self:
+        try:
+            await self.start()
+        except BaseException:
+            # unwind whatever `start` already put on the stack
+            await self.stop()
+            raise
+        return self
+
+    async def __aexit__(self, *exc) -> None:
+        await self.stop()
+
+    @abstractmethod
+    def acquire(self, session: RolloutSession) -> AbstractAsyncContextManager[Slot]:
+        """Register `session` on a server (bringing one up if needed) and yield its `Slot`;
+        free it on exit."""

verifiers-0.2.1.dev23/verifiers/v1/interception/pool.py

@@ -0,0 +1,133 @@
+"""Pools of shared interception servers, so N concurrent rollouts need ~N/multiplex
+servers + tunnels rather than one each.
+
+Behind a remote consumer each interception server needs a tunnel, and prime tunnel creation
+is rate-capped per API token — so one-tunnel-per-rollout caps how wide a remote eval (or env
+server) can fan out. Each shared `InterceptionServer` multiplexes rollouts behind one
+tunnel; the harness is unchanged, authenticating with a per-rollout secret the server routes
+by. Two shapes: `ElasticInterceptionPool` grows servers on demand (`multiplex` rollouts
+each, always prime tunnels — the only kind the framework can mint) and fits both the bounded
+eval runner and the env server's unbounded request load; `StaticInterceptionPool` is a fixed
+set of servers (each with its own tunnel choice, e.g. bring-your-own endpoints), balanced
+least-loaded.
+"""
+
+import asyncio
+import logging
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager
+from typing import Literal
+
+from pydantic import Field
+
+from verifiers.v1.interception.base import BaseInterceptionConfig, Interception, Slot
+from verifiers.v1.interception.server import (
+    InterceptionServer,
+    InterceptionServerConfig,
+)
+from verifiers.v1.interception.tunnel import PrimeTunnelConfig
+from verifiers.v1.session import RolloutSession
+
+logger = logging.getLogger(__name__)
+
+
+class StaticInterceptionPoolConfig(BaseInterceptionConfig):
+    """A fixed set of interception servers, each configured like a `server` type; rollouts
+    land on the least-loaded one. The shape for multiple bring-your-own endpoints (one
+    `custom` tunnel per server)."""
+
+    type: Literal["static"] = "static"
+    servers: list[InterceptionServerConfig] = Field(min_length=1)
+    """One entry per server, each with its own `tunnel` choice."""
+
+
+class StaticInterceptionPool(Interception):
+    """A fixed set of interception servers, all started up front; `acquire` hands a rollout
+    a slot on the least-loaded one. No capacity cap — sizing the set to the load is the
+    operator's call (it's the shape for pre-provisioned/bring-your-own endpoints)."""
+
+    def __init__(
+        self, config: StaticInterceptionPoolConfig, requires_tunnel: bool = False
+    ) -> None:
+        super().__init__()
+        self.config = config
+        self.servers = [
+            InterceptionServer(server, requires_tunnel) for server in config.servers
+        ]
+
+    async def start(self) -> None:
+        for server in self.servers:
+            await self.stack.enter_async_context(server)
+
+    @asynccontextmanager
+    async def acquire(self, session: RolloutSession) -> AsyncIterator[Slot]:
+        # min + register have no await between them, so concurrent acquires can't all
+        # land on the same "least-loaded" server.
+        server = min(self.servers, key=lambda s: s.load)
+        secret = server.register(session)
+        try:
+            yield server.base_url, secret
+        finally:
+            server.unregister(secret)
+
+
+class ElasticInterceptionPoolConfig(BaseInterceptionConfig):
+    """Interception servers grown on demand: `multiplex` rollouts share one server (and,
+    behind a remote consumer, one prime tunnel). The default."""
+
+    type: Literal["elastic"] = "elastic"
+    multiplex: int = Field(32, ge=1)
+    """Rollouts that share one interception server (and tunnel). N concurrent rollouts use
+    ~N/multiplex servers + tunnels instead of one each — key past the per-token tunnel cap.
+    1 = a server (+ tunnel) per rollout."""
+
+
+class ElasticInterceptionPool(Interception):
+    """Interception servers grown on demand: `multiplex` rollouts share one server (one
+    prime tunnel behind a remote consumer); `acquire` hands a rollout a slot on one,
+    bringing up a new server when all are at capacity."""
+
+    def __init__(
+        self,
+        config: ElasticInterceptionPoolConfig | None = None,
+        requires_tunnel: bool = False,
+    ) -> None:
+        super().__init__()
+        self.config = config or ElasticInterceptionPoolConfig()
+        self.requires_tunnel = requires_tunnel
+        self.servers: list[InterceptionServer] = []
+        self._lock = asyncio.Lock()
+
+    async def start(self) -> None:
+        pass  # servers are brought up lazily in `acquire`, on `stack`
+
+    async def _server(self) -> InterceptionServer:
+        """A server with spare capacity — reuse one under `multiplex`, else bring up a new
+        one (its own tunnel, on `stack`, torn down with the pool). The caller holds
+        `_lock`."""
+        for server in self.servers:
+            if server.load < self.config.multiplex:
+                return server
+        # Pin prime explicitly — the only tunnel kind that can be minted on demand.
+        server = InterceptionServer(
+            InterceptionServerConfig(tunnel=PrimeTunnelConfig()), self.requires_tunnel
+        )
+        await self.stack.enter_async_context(server)
+        self.servers.append(server)
+        logger.info(
+            "interception pool: %d server(s), multiplex=%d",
+            len(self.servers),
+            self.config.multiplex,
+        )
+        return server
+
+    @asynccontextmanager
+    async def acquire(self, session: RolloutSession) -> AsyncIterator[Slot]:
+        # Register under the lock so concurrent acquires see each other's load.
+        async with self._lock:
+            server = await self._server()
+            secret = server.register(session)
+        try:
+            yield server.base_url, secret
+        finally:
+            server.unregister(secret)