verifiers 0.3.2.dev79__tar.gz → 0.3.2.dev81__tar.gz

{verifiers-0.3.2.dev79 → verifiers-0.3.2.dev81}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.3.2.dev79
+Version: 0.3.2.dev81
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.3.2.dev79 → verifiers-0.3.2.dev81}/tests/v1/conftest.py

@@ -55,7 +55,7 @@ CI_MODEL = "openai/gpt-5.6-luna"
 
 
 @pytest.fixture
-def harness_runtime(request) -> str:
+def harness_runtime(request) -> str | dict:
     return request.param
 
 

{verifiers-0.3.2.dev79 → verifiers-0.3.2.dev81}/tests/v1/test_e2e.py

@@ -53,6 +53,12 @@ AGENTIC_PLACEMENTS = [
     pair("hermes-agent", "docker", "hermes-agent-harness-in-docker"),
     pair("bash", "prime", "bash-harness-in-prime"),
     pair("bash", "modal", "bash-harness-in-modal"),
+    pytest.param(
+        "bash",
+        {"type": "modal", "allow": []},
+        marks=[mark.bash, mark.modal],
+        id="bash-harness-in-modal-framework-only",
+    ),
 ]
 
 # The scripted user runs in the eval process itself (no placement axis); the harness
@@ -417,7 +423,9 @@ async def test_agentic(run_v1, harness, harness_runtime, tmp_path):
     (trace,) = await run_v1(
         "echo-agentic-v1",
         harness=harness,
-        runtime={"type": harness_runtime},
+        runtime=harness_runtime
+        if isinstance(harness_runtime, dict)
+        else {"type": harness_runtime},
         output_dir=tmp_path,
         max_turns=10,
         max_tokens=8192,

{verifiers-0.3.2.dev79 → verifiers-0.3.2.dev81}/verifiers/v1/runtimes/base.py

@@ -24,10 +24,11 @@ from verifiers.v1.utils.aio import run_shielded
 
 logger = logging.getLogger(__name__)
 
-# Ensure the latest `uv` is available for our PEP 723 scripts: prefer pip on Python images,
-# then fall back to the standalone installer (curl/wget), installing curl + CA certs when a
-# bare image has no downloader. Both paths install to ~/.local/bin, which we prepend to PATH.
-# (Needs network + one of pip / curl / wget / apt-get / apk.)
+# Ensure `uv` is available for our PEP 723 scripts: keep one already on PATH (an image that
+# pre-installs it, or an earlier rollout's install on the same box); otherwise prefer pip on
+# Python images, then fall back to the standalone installer (curl/wget), installing curl + CA
+# certs when a bare image has no downloader. Both install paths land in ~/.local/bin, which we
+# prepend to PATH first. (Installing needs network + one of pip / curl / wget / apt-get / apk.)
 _INSTALL_CURL = (  # only when the image has no downloader; needs a known package manager
     "{ command -v curl >/dev/null 2>&1 || command -v wget >/dev/null 2>&1; } "
     "|| { apt-get update -qq && apt-get install -y -qq curl ca-certificates; } "
@@ -39,7 +40,8 @@ _DOWNLOAD_UV = (
 )
 _ENSURE_UV = (
     'export PATH="$HOME/.local/bin:$PATH" UV_INSTALL_DIR="$HOME/.local/bin"; '
-    "pip install -q -U --user uv 2>/dev/null "
+    "command -v uv >/dev/null 2>&1 "
+    "|| pip install -q -U --user uv 2>/dev/null "
     f"|| {{ {_INSTALL_CURL}; {_DOWNLOAD_UV}; }}"
 )
 

{verifiers-0.3.2.dev79 → verifiers-0.3.2.dev81}/verifiers/v1/runtimes/modal.py

@@ -10,14 +10,18 @@ program in the sandbox reaching a host service) is the shared host-side `Tunnel`
 import asyncio
 import contextlib
 import logging
+import re
 import shlex
 import uuid
 from collections.abc import AsyncIterator
+from ipaddress import ip_address
 from pathlib import PurePosixPath
 from typing import ClassVar, Literal
+from urllib.parse import urlsplit
 
-from pydantic_config import BaseConfig
+from pydantic import model_validator
 
+from verifiers.v1.configs.runtime import NetworkPolicyConfig
 from verifiers.v1.errors import SandboxError
 from verifiers.v1.runtimes.base import (
     SERVICE_PORT,
@@ -36,12 +40,49 @@ logger = logging.getLogger(__name__)
 _APP_NAME = "verifiers-v1"
 
 
-class ModalConfig(BaseConfig):
+def _egress_domain(rule: str, *, framework: bool = False) -> str | None:
+    """Translate only domains Modal can filter without broadening the rule."""
+    parsed = urlsplit(rule if "://" in rule else f"//{rule}")
+    host = (parsed.hostname or "").lower().rstrip(".")
+    try:
+        address = ip_address(host)
+    except ValueError:
+        address = None
+    # Colocated tools are container-local; they need no external network grant.
+    if (
+        framework
+        and parsed.scheme in ("http", "https")
+        and (host == "localhost" or (address is not None and address.is_loopback))
+    ):
+        return None
+    if (
+        parsed.scheme not in (("https",) if framework else ("", "https"))
+        or parsed.port not in (None, 443)
+        or parsed.username is not None
+        or parsed.password is not None
+        or address is not None
+        or not re.fullmatch(
+            r"(?:\*\.)?(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)*"
+            r"[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?",
+            host,
+        )
+        or (framework and "*" in host)
+    ):
+        raise ValueError(
+            f"Modal egress rule {rule!r} is unsupported; use DNS names or HTTPS "
+            "origins on port 443 (only a leading *. wildcard is supported)"
+        )
+    return host
+
+
+class ModalConfig(NetworkPolicyConfig):
     type: Literal["modal"] = "modal"
     image: str = "python:3.11-slim"
     workdir: str | None = None
     """Working directory override; None uses the task's workdir, or /app."""
     network_access: bool = True
+    """Allow network access at creation. False blocks all egress, including setup;
+    it cannot be combined with execution-time allow/block policies."""
     region: str | None = None
     """Region to provision in (None = provider-chosen)."""
     # TaskData.resources uses these units; non-default runtime config values take precedence.
@@ -58,6 +99,22 @@ class ModalConfig(BaseConfig):
     """Pace sandbox creation to this many per second, enforced user-wide across every
     env-server worker process (None/<= 0 disables it)."""
 
+    @model_validator(mode="after")
+    def _validate_egress(self) -> "ModalConfig":
+        if not self.network_restricted:
+            return self
+        if not self.network_access:
+            raise ValueError(
+                "Modal allow/block policies require network_access=true for trusted setup"
+            )
+        if self.allow == ["*"]:
+            raise ValueError(
+                "Modal does not support egress deny lists; use an allowlist or allow=[]"
+            )
+        for rule in self.allow:
+            _egress_domain(rule)
+        return self
+
 
 class ModalRuntimeInfo(ModalConfig, BaseRuntimeInfo):
     pass
@@ -124,6 +181,14 @@ class ModalRuntime(Runtime):
                 "ModalRuntime requires the Modal SDK; install `verifiers[modal]`."
             ) from e
 
+        if self.network_restricted and not hasattr(
+            modal.Sandbox, "_experimental_set_outbound_network_policy"
+        ):
+            raise SandboxError(
+                "Modal execution-time network policies require a Modal SDK with "
+                "_experimental_set_outbound_network_policy; upgrade verifiers[modal]"
+            )
+
         try:
             app = await modal.App.lookup.aio(_APP_NAME, create_if_missing=True)
             async with (
@@ -152,6 +217,16 @@ class ModalRuntime(Runtime):
         """
         import modal
 
+        # Modal requires both allowlist types at creation before they can be updated.
+        # Trusted setup runs open; prepare_execution removes the broad CIDR grant.
+        network = (
+            {
+                "outbound_domain_allowlist": ["*"],
+                "outbound_cidr_allowlist": ["0.0.0.0/0"],
+            }
+            if self.network_restricted
+            else {}
+        )
         self._sandbox = await modal.Sandbox.create.aio(
             "sleep",
             "infinity",  # keep-alive entrypoint; the harness runs via `exec`
@@ -168,10 +243,53 @@ class ModalRuntime(Runtime):
             gpu=self.config.gpu,
             region=self.config.region,
             block_network=not self.config.network_access,
+            **network,
             timeout=24 * 60 * 60,  # Maximum lifetime of any sandbox.
             encrypted_ports=[SERVICE_PORT],
         )
 
+    async def prepare_execution(self, routes: list[str] | None) -> None:
+        """Apply TLS domain filtering after setup, retaining framework endpoints.
+
+        Modal filters TLS SNI on port 443, not HTTP Host headers or URL paths.
+        This inherits Modal's domain-fronting limitations on shared TLS endpoints.
+        """
+        if not self.network_restricted:
+            return
+        try:
+            if routes is None:
+                domains, cidrs = ["*"], ["0.0.0.0/0"]
+            else:
+                domains = list(
+                    dict.fromkeys(
+                        domain
+                        for domain in [
+                            *(
+                                _egress_domain(route, framework=True)
+                                for route in routes
+                            ),
+                            *(_egress_domain(rule) for rule in self.config.allow),
+                        ]
+                        if domain is not None
+                    )
+                )
+                cidrs = []
+            # Always send both lists: leaving the setup CIDR grant would bypass domains.
+            # The awaited RPC applies the policy and closes newly disallowed connections.
+            async with asyncio.timeout(60):
+                await self._sandbox._experimental_set_outbound_network_policy.aio(
+                    outbound_domain_allowlist=domains,
+                    outbound_cidr_allowlist=cidrs,
+                )
+        except Exception as e:
+            raise SandboxError(f"modal egress policy failed: {e}") from e
+        logger.info(
+            "modal: egress policy applied on sandbox %s (domains=%s cidrs=%s)",
+            self.info.id,
+            domains,
+            cidrs,
+        )
+
     async def expose(self, port: int) -> str | None:
         # Publish a server hosted IN the sandbox: Modal forwards `port` (named via
         # `encrypted_ports` at creation) to a public URL, read back from `sandbox.tunnels()`.