verifiers 0.2.2.dev14__tar.gz → 0.2.2.dev16__tar.gz

{verifiers-0.2.2.dev14 → verifiers-0.2.2.dev16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.2.dev14
+Version: 0.2.2.dev16
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.2.2.dev14 → verifiers-0.2.2.dev16}/verifiers/envs/experimental/composable/README.md

@@ -115,7 +115,7 @@ Each taskset provides its own rubric via `get_rubric()`. The rubric owns **all s
 
 Use `keep_sandbox_for_scoring=True` on the environment so the sandbox stays alive after the rollout for the rubric to use.
 
-```
+```text
 Rollout completes → sandbox kept alive → rubric.score_rollout() runs tests
     → rubric.cleanup() deletes sandbox
 ```

{verifiers-0.2.2.dev14 → verifiers-0.2.2.dev16}/verifiers/envs/integrations/README.md

@@ -139,7 +139,7 @@ Drop-in adapter for [OpenEnv](https://github.com/meta-pytorch/OpenEnv) environme
 
 The Verifiers adapter uses OpenEnv's public async clients. The bundled OpenEnv project under `proj/` declares its own server dependencies for the sandbox image.
 
-### Quick Start
+### OpenEnv Quick Start
 
 Initialize an OpenEnv environment with the template:
 

{verifiers-0.2.2.dev14 → verifiers-0.2.2.dev16}/verifiers/v1/agent.py

@@ -26,6 +26,7 @@ from verifiers.v1.mcp import SharedToolServer
 from verifiers.v1.retries import RetryConfig, backoff, trace_should_retry
 from verifiers.v1.rollout import RolloutRun
 from verifiers.v1.runtimes import (
+    DockerConfig,
     Runtime,
     RuntimeConfig,
     SubprocessConfig,
@@ -93,11 +94,37 @@ class AgentConfig(BaseConfig):
         return data
 
 
-def _check_borrowed_placement(task: Task, runtime: Runtime) -> None:
+def _check_borrowed_placement(
+    task: Task, runtime: Runtime, base_config: RuntimeConfig
+) -> None:
     """A borrowed box is never re-provisioned, so a task's placement fields can't
-    be honored. A task `image` on a subprocess box raises (a wiring bug — it goes
-    to the caller, not the trace); a container box whose image differs only warns,
-    since placing a run into an existing world is the point of borrowing."""
+    be honored. Reject requirements that cannot be applied to the running box; an
+    image mismatch on a container only warns, since sharing its world is the point."""
+    task_policy = "*" not in task.data.network_allow or bool(task.data.network_block)
+    base_docker = base_config if isinstance(base_config, DockerConfig) else None
+    if task_policy or (base_docker is not None and base_docker.network_isolated):
+        config = runtime.config
+        if not isinstance(config, DockerConfig):
+            raise ValueError(
+                f"task {task.data.idx!r} requires a Docker URL network policy, but "
+                f"borrowed runtime {runtime.name!r} is not Docker-backed; use "
+                "agent.provision(task)"
+            )
+        policy_base = (
+            base_docker if base_docker is not None else DockerConfig(allow=["*"])
+        )
+        expected = resolve_runtime_config(policy_base, task)
+        assert isinstance(expected, DockerConfig)
+        # Do not inherit extra destinations from a box provisioned for another task.
+        if set(config.allow) != set(expected.allow) or set(config.block) != set(
+            expected.block
+        ):
+            raise ValueError(
+                f"task {task.data.idx!r} requires allow={expected.allow!r} and "
+                f"block={expected.block!r}, but borrowed runtime {runtime.name!r} "
+                f"has allow={config.allow!r} and block={config.block!r}; use "
+                "agent.provision(task)"
+            )
     if task.data.image is None:
         return
     if isinstance(runtime.config, SubprocessConfig):
@@ -299,7 +326,7 @@ class Agent:
     ) -> dict:
         """Resolve one run's runtime config, pairing checks, timeouts, interception."""
         if runtime is not None:
-            _check_borrowed_placement(task, runtime)
+            _check_borrowed_placement(task, runtime, self.runtime_config)
             runtime_config = runtime.config
             run_is_local = runtime.is_local
         else:

{verifiers-0.2.2.dev14 → verifiers-0.2.2.dev16}/verifiers/v1/cli/debug.py

@@ -230,6 +230,7 @@ async def debug_task(task: Task, config: DebugConfig) -> tuple[Trace, bool]:
             invoke(task.setup, {"trace": trace, "runtime": runtime}),
             setup_timeout,
         )
+        await runtime.prepare_execution([])
         trace.timing.setup.end = time.time()
 
         trace.timing.generation.start = time.time()

{verifiers-0.2.2.dev14 → verifiers-0.2.2.dev16}/verifiers/v1/interception/__init__.py

@@ -34,13 +34,10 @@ def requires_tunnel(
     server_configs: Iterable[BaseConfig] = (),
     shared: "Iterable[SharedToolServer]" = (),
 ) -> bool:
-    """Whether the interception must be exposed via a tunnel — some consumer is off the
-    host network: the harness itself, a live `shared` server in a remote runtime, or a
-    tool/user server config placing one there (each reaches the `/state` channel from
-    its own runtime). Skipped as non-consumers: a `colocated` server (shares the
-    harness's runtime, covered by `harness_is_local`), a config-`url` server (external —
-    it connects out), and an `external` shared server (outside the state machinery
-    entirely). False means every consumer reaches the server at localhost."""
+    """Whether interception needs a public tunnel because some consumer cannot use a
+    host-local URL: the harness itself, a live `shared` server in a remote runtime, or a
+    tool/user server placed there. Colocated servers, configured URLs, and external shared
+    servers add no consumer. False means local URL translation is sufficient."""
     if not harness_is_local:
         return True
     if any(not s.external and not s.local for s in shared):
@@ -57,10 +54,8 @@ def make_interception(
     config: InterceptionConfig, *, requires_tunnel: bool
 ) -> Interception:
     """The interception for a config, picked by type (the host-side counterpart to
-    `make_runtime`). With `requires_tunnel` (some consumer is off the host network — see
-    the `requires_tunnel` util) each server is exposed via its configured tunnel; without
-    it they get none and are reached at localhost. The caller computes it (see
-    `Env._requires_tunnel`)."""
+    `make_runtime`). With `requires_tunnel`, each server is exposed through its configured
+    tunnel; otherwise it remains on host loopback. The caller computes this requirement."""
     if isinstance(config, InterceptionServerConfig):
         return InterceptionServer(config, requires_tunnel)
     if isinstance(config, StaticInterceptionPoolConfig):

{verifiers-0.2.2.dev14 → verifiers-0.2.2.dev16}/verifiers/v1/interception/server.py

@@ -121,11 +121,9 @@ class InterceptionServerConfig(BaseInterceptionConfig):
 
 class InterceptionServer(Interception):
     """A server that proxies model calls for one or more rollouts — and is itself the
-    single-server `Interception` (the pools compose several of these). With
-    `requires_tunnel` (some consumer is off the host network) it mints its configured
-    tunnel; on `start` it then binds where the tunnel says (`bind_host`/`bind_port`) and
-    sets `base_url` — the one URL every consumer reaches it at — to the tunnel's public
-    URL. Without, every consumer is on the host network: it binds loopback, tunnel-free."""
+    single-server `Interception` (the pools compose several of these). When a consumer
+    needs a public URL, it mints the configured tunnel and binds where that tunnel says;
+    otherwise it stays on host loopback."""
 
     def __init__(
         self,
@@ -203,9 +201,8 @@ class InterceptionServer(Interception):
         self.runner = web.AppRunner(app)
         await self.runner.setup()
         self.stack.push_async_callback(self.runner.cleanup)
-        # No tunnel → every consumer shares the host network: bind loopback on any ephemeral
-        # port. Otherwise the tunnel says where to bind for it to reach the port, and
-        # `expose` publishes it.
+        # Without a tunnel, local URL translation reaches an ephemeral loopback port.
+        # Otherwise the tunnel determines the bind address and publishes it.
         if self.tunnel is None:
             self.host, bind_port = "127.0.0.1", 0
         else:

{verifiers-0.2.2.dev14 → verifiers-0.2.2.dev16}/verifiers/v1/interception/tunnel/base.py

@@ -1,12 +1,9 @@
 """The Tunnel contract: expose a host interception port to a remote consumer.
 
-The interception server runs on the host. A *local* consumer (host network) reaches it at
-localhost — no tunnel, whatever the config. A *remote* one needs the port published outward:
-that's a `Tunnel`. It says where the server must bind for the tunnel to reach it
-(`bind_host`/`bind_port`) and `expose`s the bound port as a public URL. So a tunnel knows
-nothing about locality — the caller (`InterceptionServer.start`) uses it only in the remote
-case. The host-side counterpart to a `Runtime`: `Runtime.expose` publishes a port *inside* a
-sandbox; `Tunnel.expose` a *host* port.
+The interception server runs on the host. Local runtimes use a host-local URL directly or
+through their runtime translation; remote runtimes need the port published outward. A
+`Tunnel` supplies the bind address and exposes that host port as a public URL. It is the
+host-side counterpart to `Runtime.expose`, which publishes a port inside a sandbox.
 """
 
 import contextlib

{verifiers-0.2.2.dev14 → verifiers-0.2.2.dev16}/verifiers/v1/mcp/launch.py

@@ -23,6 +23,7 @@ from verifiers.v1.errors import RolloutError, ToolsetError, UserError
 from verifiers.v1.interception.tunnel import PrimeTunnel
 from verifiers.v1.mcp.server import STATE_SECRET_PARAM, STATE_URL_PARAM, ServerBase
 from verifiers.v1.runtimes import (
+    DockerConfig,
     Runtime,
     make_runtime,
     runtime_is_local,
@@ -238,18 +239,18 @@ async def reachable_url(
     """Yield the URL a consumer uses to reach the server at (`service`, `port`), over two
     primitives: `Runtime.expose` (publish a port out of a sandbox) and a host `Tunnel` (reach
     into the host from a remote runtime). `colocated` = the server shares the consumer's
-    runtime; `consumer_is_local` = the consumer is on the host network.
+    runtime; `consumer_is_local` = the consumer can use a host-local URL without a tunnel.
 
     - `colocated` -> localhost (same runtime, in-sandbox or host loopback);
     - the server runs in a remote sandbox -> its own published URL (`expose`), reachable anywhere;
-    - else it's on the host network -> localhost to a local consumer, a host tunnel to a remote one."""
+    - else it's host-local -> localhost to a local consumer, a host tunnel to a remote one."""
     if colocated:
         yield f"http://127.0.0.1:{port}"
     elif not service.is_local:  # in a remote sandbox → it publishes its own port
         yield await service.expose(port)
-    elif consumer_is_local:  # host network, local consumer → localhost, no tunnel
+    elif consumer_is_local:  # local consumer → localhost, no public tunnel
         yield f"http://127.0.0.1:{port}"
-    else:  # host network, remote consumer → a host tunnel publishes the port outward
+    else:  # remote consumer → a host tunnel publishes the port outward
         async with PrimeTunnel().expose(port) as url:
             yield url
 
@@ -265,8 +266,33 @@ async def serve(
     state_base: str | None = None,
 ):
     cfg = server.config
+    colocated = getattr(cfg, "colocated", False)
     async with contextlib.AsyncExitStack() as stack:
-        if getattr(cfg, "colocated", False) and harness_runtime is not None:
+        # Colocated servers inherit the harness cut. A separately provisioned filtered
+        # Docker server has neither that lifecycle nor a published port after isolation;
+        # reject it instead of silently leaving its requested policy unenforced.
+        if (
+            isinstance(cfg.runtime, DockerConfig)
+            and cfg.runtime.network_isolated
+            and not (colocated and harness_runtime is not None)
+        ):
+            raise ToolsetError(
+                "Docker network policies are supported on the harness runtime; "
+                f"server {server.server_name!r} must be colocated or use an "
+                "unrestricted Docker runtime"
+            )
+        if (
+            harness_runtime is not None
+            and harness_runtime.network_isolated
+            and colocated
+            and for_host
+        ):
+            raise UserError(
+                "a colocated user simulator is unreachable from the host when "
+                "the harness uses a Docker network policy; use a local "
+                "(subprocess) user simulator"
+            )
+        if colocated and harness_runtime is not None:
             runtime = harness_runtime
         else:
             runtime = make_runtime(cfg.runtime)
@@ -307,7 +333,9 @@ async def serve(
                 runtime, port, colocated=colocated, consumer_is_local=consumer_is_local
             )
         )
-        if not for_host and not colocated and harness_runtime is not None:
+        if colocated and harness_runtime is not None and runtime.network_isolated:
+            base = base.replace("127.0.0.1", "localhost", 1)
+        elif not colocated and not for_host and harness_runtime is not None:
             base = harness_runtime.host_url(base)
         yield f"{base.rstrip('/')}/mcp"
 
@@ -406,7 +434,7 @@ async def serve_tools(
                 # Not ours: a pre-existing endpoint with no vf state channel. Pass the URL
                 # through bare — a state tag would be useless, and the per-rollout secret
                 # must not ride the query string to a third-party host.
-                urls[name] = server.url
+                urls[name] = harness_runtime.host_url(server.url)
                 logger.info("tool server '%s' (shared, external): %s", name, server.url)
                 continue
             url = harness_runtime.host_url(server.url) if server.local else server.url
@@ -422,7 +450,7 @@ async def serve_tools(
                 )
             cfg = toolset.config
             if cfg.url:
-                urls[name] = cfg.url
+                urls[name] = harness_runtime.host_url(cfg.url)
                 logger.info("tool server '%s' (remote): %s", name, cfg.url)
             else:
                 urls[name] = await stack.enter_async_context(

{verifiers-0.2.2.dev14 → verifiers-0.2.2.dev16}/verifiers/v1/rollout.py

@@ -191,6 +191,7 @@ class RolloutRun:
         try:
             if self._owns_runtime:
                 await runtime.start()
+            await runtime.prepare_setup()
             now = time.time()
             self.trace.timing.boot.end = now
             self.trace.timing.setup.start = now
@@ -251,6 +252,9 @@ class RolloutRun:
                     "conversation; set task.prompt or declare a simulator "
                     "class on Task.user"
                 )
+            # Setup and service provisioning are complete. Apply the runtime's
+            # execution policy while preserving the framework routes the agent uses.
+            await runtime.prepare_execution([self._endpoint, *self._urls.values()])
         except Exception as e:
             self.fail(e)
             return False

{verifiers-0.2.2.dev14 → verifiers-0.2.2.dev16}/verifiers/v1/runtimes/__init__.py

@@ -45,9 +45,8 @@ def make_runtime(config: RuntimeConfig, name: str | None = None) -> Runtime:
 
 
 def runtime_is_local(config: RuntimeConfig) -> bool:
-    """Whether a runtime of this config shares the host network (so a program inside it reaches a
-    host service at localhost, no tunnel) — read off the runtime class, without provisioning one.
-    The interception / tool serving use it to decide whether to tunnel their host port."""
+    """Whether a runtime of this config exchanges host-local URLs without a public
+    tunnel, read off the runtime class without provisioning one."""
     return _runtime_cls(config).is_local
 
 

{verifiers-0.2.2.dev14 → verifiers-0.2.2.dev16}/verifiers/v1/runtimes/base.py

@@ -103,10 +103,9 @@ class BaseRuntimeInfo(BaseConfig):
 
 class Runtime(ABC):
     is_local: ClassVar[bool] = True
-    """Whether this runtime shares the host network — a program inside it reaches a host service
-    at localhost (no tunnel) and a service inside it is reachable at localhost. True for
-    subprocess / docker(--network host); remote runtimes (modal/prime) override to False (they
-    need a tunnel each way: a host `Tunnel` (interception.tunnel) inward, `expose` outward)."""
+    """Whether this runtime exchanges host-local URLs without a public tunnel. True for
+    subprocess and Docker (directly or through Docker's policy proxy); remote runtimes
+    override to False and use a host `Tunnel` inward plus `expose` outward."""
 
     info: BaseRuntimeInfo
 
@@ -243,19 +242,30 @@ class Runtime(ABC):
         """The URL a program inside this runtime uses to reach a host-bound `url`."""
         return url
 
+    async def prepare_setup(self) -> None:
+        """Claim the runtime for trusted setup; restricted runtimes may reject reuse."""
+
+    async def prepare_execution(self, routes: list[str]) -> None:
+        """Last setup step, right before the agent starts. Restricted runtimes enforce
+        their policy here while keeping the interception and MCP `routes` reachable."""
+
+    @property
+    def network_isolated(self) -> bool:
+        """Whether the agent phase uses filtered networking (see `prepare_execution`)."""
+        return False
+
     @property
     def published_port(self) -> int | None:
         """A fixed port this runtime exposes to the outside at startup, declared up front to the
         provider (Modal forwards only ports named at `Sandbox.create`). When set, a server placed
         here binds it instead of a host-chosen free port, and `expose` returns its public URL.
-        `None` for host-networked runtimes (subprocess/docker), which pick a free port and are
-        reached over the shared host network."""
+        `None` for local runtimes (subprocess/docker), which pick a free port."""
         return None
 
     async def expose(self, port: int) -> str | None:
         """Publish a port running *inside this runtime* to a URL reachable from the host/outside,
-        or None when local (it's on the host network — reach it at localhost). A remote runtime
-        overrides this with the provider's native port exposure (modal `tunnels()`, prime
-        `client.expose`), torn down with the sandbox in `stop()`. The reverse of a host `Tunnel`
-        (interception.tunnel, which reaches a host port from inside a runtime)."""
+        or None when local. A remote runtime overrides this with the provider's native port
+        exposure (modal `tunnels()`, prime `client.expose`), torn down with the sandbox in
+        `stop()`. The reverse of a host `Tunnel` (interception.tunnel, which reaches a host
+        port from inside a runtime)."""
         return None