verifiers 0.2.2.dev22__tar.gz → 0.2.2.dev24__tar.gz

{verifiers-0.2.2.dev22 → verifiers-0.2.2.dev24}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.2.dev22
+Version: 0.2.2.dev24
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers
@@ -35,7 +35,7 @@ Requires-Dist: numpy>=2.1.0
 Requires-Dist: openai-agents>=0.8.2
 Requires-Dist: openai>=2.9.0
 Requires-Dist: prime-pydantic-config[toml]>=0.4.2
-Requires-Dist: prime-sandboxes>=0.2.31
+Requires-Dist: prime-sandboxes>=0.2.33
 Requires-Dist: prime-tunnel>=0.1.8
 Requires-Dist: pydantic>=2.12.3
 Requires-Dist: pyzmq>=27.1.0

{verifiers-0.2.2.dev22 → verifiers-0.2.2.dev24}/pyproject.toml

@@ -35,7 +35,7 @@ dependencies = [
     "openai>=2.9.0",
     "openai-agents>=0.8.2",
     "prime-tunnel>=0.1.8",
-    "prime-sandboxes>=0.2.31",
+    "prime-sandboxes>=0.2.33",
     "pydantic>=2.12.3",
     "requests",
     "rich>=11.0.0",

{verifiers-0.2.2.dev22 → verifiers-0.2.2.dev24}/tests/v1/fixtures/pyproject.toml

@@ -15,6 +15,7 @@ build-backend = "hatchling.build"
 [tool.hatch.build]
 include = [
     "counter_tool_v1.py",
+    "echo_acp_resume_v1.py",
     "echo_tool_v1.py",
     "tool_response_image_v1.py",
     "pyproject.toml",

{verifiers-0.2.2.dev22 → verifiers-0.2.2.dev24}/verifiers/v1/agent.py

@@ -30,7 +30,7 @@ from verifiers.v1.mcp import SharedToolServer
 from verifiers.v1.retries import RetryConfig, backoff, trace_should_retry
 from verifiers.v1.rollout import RolloutRun, _as_messages
 from verifiers.v1.runtimes import (
-    DockerConfig,
+    NetworkPolicyConfig,
     Runtime,
     RuntimeConfig,
     SubprocessConfig,
@@ -114,20 +114,26 @@ def _check_borrowed_placement(
     be honored. Reject requirements that cannot be applied to the running box; an
     image mismatch on a container only warns, since sharing its world is the point."""
     task_policy = "*" not in task.data.network_allow or bool(task.data.network_block)
-    base_docker = base_config if isinstance(base_config, DockerConfig) else None
-    if task_policy or (base_docker is not None and base_docker.network_isolated):
+    base_policy = base_config if isinstance(base_config, NetworkPolicyConfig) else None
+    if task_policy or (base_policy is not None and base_policy.network_restricted):
         config = runtime.config
-        if not isinstance(config, DockerConfig):
+        if not isinstance(config, NetworkPolicyConfig):
             raise ValueError(
-                f"task {task.data.idx!r} requires a Docker URL network policy, but "
-                f"borrowed runtime {runtime.name!r} is not Docker-backed; use "
+                f"task {task.data.idx!r} requires a framework-aware network policy, "
+                f"but borrowed runtime {runtime.name!r} does not support one; use "
                 "agent.provision(task)"
             )
-        policy_base = (
-            base_docker if base_docker is not None else DockerConfig(allow=["*"])
+        if base_policy is not None and type(config) is not type(base_policy):
+            raise ValueError(
+                f"the configured {base_policy.type} network policy cannot be applied "
+                f"to borrowed {config.type} runtime {runtime.name!r}; use "
+                "agent.provision(task)"
+            )
+        policy_base = base_policy or config.model_copy(
+            update={"allow": ["*"], "block": []}
         )
         expected = resolve_runtime_config(policy_base, task)
-        assert isinstance(expected, DockerConfig)
+        assert isinstance(expected, NetworkPolicyConfig)
         # Do not inherit extra destinations from a box provisioned for another task.
         if set(config.allow) != set(expected.allow) or set(config.block) != set(
             expected.block

{verifiers-0.2.2.dev22 → verifiers-0.2.2.dev24}/verifiers/v1/mcp/launch.py

@@ -20,7 +20,7 @@ from verifiers.v1.errors import ToolsetError
 from verifiers.v1.interception.tunnel import PrimeTunnel
 from verifiers.v1.mcp.server import STATE_SECRET_PARAM, STATE_URL_PARAM, ServerBase
 from verifiers.v1.runtimes import (
-    DockerConfig,
+    NetworkPolicyConfig,
     Runtime,
     make_runtime,
     runtime_is_local,
@@ -253,17 +253,17 @@ async def serve(
     colocated = getattr(cfg, "colocated", False)
     async with contextlib.AsyncExitStack() as stack:
         # Colocated servers inherit the harness cut. A separately provisioned filtered
-        # Docker server has neither that lifecycle nor a published port after isolation;
+        # server has neither that lifecycle nor a published port after isolation;
         # reject it instead of silently leaving its requested policy unenforced.
         if (
-            isinstance(cfg.runtime, DockerConfig)
-            and cfg.runtime.network_isolated
+            isinstance(cfg.runtime, NetworkPolicyConfig)
+            and cfg.runtime.network_restricted
             and not (colocated and harness_runtime is not None)
         ):
             raise ToolsetError(
-                "Docker network policies are supported on the harness runtime; "
+                "Runtime network policies are supported on the harness runtime; "
                 f"server {server.server_name!r} must be colocated or use an "
-                "unrestricted Docker runtime"
+                "unrestricted runtime"
             )
         if colocated and harness_runtime is not None:
             runtime = harness_runtime
@@ -303,7 +303,7 @@ async def serve(
                 runtime, port, colocated=colocated, consumer_is_local=consumer_is_local
             )
         )
-        if colocated and harness_runtime is not None and runtime.network_isolated:
+        if colocated and harness_runtime is not None and runtime.network_restricted:
             base = base.replace("127.0.0.1", "localhost", 1)
         elif not colocated and harness_runtime is not None:
             base = harness_runtime.host_url(base)

{verifiers-0.2.2.dev22 → verifiers-0.2.2.dev24}/verifiers/v1/push.py

@@ -6,6 +6,7 @@ contract as `prime eval push`, done inline at the end of a run. Auth + base URL
 come from `$PRIME_API_KEY` / `~/.prime/config.json`.
 """
 
+import json
 import logging
 import os
 from dataclasses import dataclass
@@ -22,6 +23,8 @@ logger = logging.getLogger(__name__)
 
 DEFAULT_API_URL = "https://api.primeintellect.ai"
 DEFAULT_FRONTEND_URL = "https://app.primeintellect.ai"
+# Repeated /samples posts append; match the Prime Evals client's request ceiling.
+_MAX_SAMPLES_PAYLOAD_BYTES = 25 * 1024 * 1024
 
 
 @dataclass
@@ -200,6 +203,36 @@ def push_traces(
     # The run is done and its results saved; a network blip here must not crash it
     # — log and skip the upload instead.
     try:
+        batches: list[list[dict[str, Any]]] = []
+        batch: list[dict[str, Any]] = []
+        payload_bytes = len(b'{"samples":[]}')
+        for i, sample in enumerate(samples):
+            sample_bytes = len(
+                json.dumps(
+                    sample,
+                    ensure_ascii=False,
+                    separators=(",", ":"),
+                    allow_nan=False,
+                ).encode("utf-8")
+            )
+            sample_payload_bytes = len(b'{"samples":[]}') + sample_bytes
+            if sample_payload_bytes > _MAX_SAMPLES_PAYLOAD_BYTES:
+                raise ValueError(
+                    f"sample {i} is too large to upload "
+                    f"({sample_payload_bytes} > "
+                    f"{_MAX_SAMPLES_PAYLOAD_BYTES} bytes)"
+                )
+            next_payload_bytes = payload_bytes + (1 if batch else 0) + sample_bytes
+            if batch and next_payload_bytes > _MAX_SAMPLES_PAYLOAD_BYTES:
+                batches.append(batch)
+                batch = []
+                payload_bytes = len(b'{"samples":[]}')
+                next_payload_bytes = payload_bytes + sample_bytes
+            batch.append(sample)
+            payload_bytes = next_payload_bytes
+        if batch or not samples:
+            batches.append(batch)
+
         with httpx.Client(headers=headers, timeout=300.0) as client:
 
             def post(path: str, body: dict) -> dict:
@@ -224,7 +257,18 @@ def push_traces(
                     **team,
                 },
             )["evaluation_id"]
-            post(f"/evaluations/{eval_id}/samples", {"samples": samples})
+            for batch in batches:
+                body = json.dumps(
+                    {"samples": batch},
+                    ensure_ascii=False,
+                    separators=(",", ":"),
+                    allow_nan=False,
+                ).encode("utf-8")
+                resp = client.post(
+                    f"{api}/evaluations/{eval_id}/samples",
+                    content=body,
+                )
+                resp.raise_for_status()
             post(f"/evaluations/{eval_id}/finalize", {"metrics": metrics})
     except Exception as e:
         logger.warning("--push: upload failed (%s: %s); skipping", type(e).__name__, e)

{verifiers-0.2.2.dev22 → verifiers-0.2.2.dev24}/verifiers/v1/runtimes/__init__.py

@@ -4,6 +4,7 @@ from pydantic import Field
 
 from verifiers.v1.runtimes.base import (
     BaseRuntimeInfo,
+    NetworkPolicyConfig,
     ProgramResult,
     Runtime,
     register,
@@ -56,6 +57,7 @@ __all__ = [
     "RuntimeConfig",
     "RuntimeInfo",
     "BaseRuntimeInfo",
+    "NetworkPolicyConfig",
     "make_runtime",
     "runtime_is_local",
     "SubprocessConfig",

{verifiers-0.2.2.dev22 → verifiers-0.2.2.dev24}/verifiers/v1/runtimes/base.py

@@ -11,10 +11,11 @@ import weakref
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
 from pathlib import PurePosixPath
-from typing import ClassVar
+from typing import ClassVar, Self
 
 from pydantic_config import BaseConfig
 
+from verifiers.v1.errors import SandboxError
 from verifiers.v1.utils.aio import run_shielded
 
 logger = logging.getLogger(__name__)
@@ -94,6 +95,33 @@ def cleanup_at_exit() -> None:
             runtime.cleanup()
 
 
+class NetworkPolicyConfig(BaseConfig):
+    """Shared execution-time policy surface for runtimes that support it."""
+
+    allow: list[str] = ["*"]
+    """Destinations allowed during execution; `*` leaves egress unrestricted."""
+    block: list[str] = []
+    """Destinations denied during execution."""
+
+    @property
+    def network_restricted(self) -> bool:
+        return "*" not in self.allow or bool(self.block)
+
+    def with_task_network_policy(self, allow: list[str], block: list[str]) -> Self:
+        if "*" not in allow:
+            allow = (
+                allow
+                if "*" in self.allow
+                else list(dict.fromkeys([*allow, *self.allow]))
+            )
+        else:
+            allow = self.allow
+        block = list(dict.fromkeys([*block, *self.block]))
+        return type(self).model_validate(
+            {**self.model_dump(), "allow": allow, "block": block}
+        )
+
+
 class BaseRuntimeInfo(BaseConfig):
     id: str | None = None
     borrowed: bool = False
@@ -113,6 +141,7 @@ class Runtime(ABC):
         self.name = name or f"vf-{uuid.uuid4().hex[:12]}"
         self._uv_interpreters: dict[str, str] = {}
         self._uv_script_locks: dict[str, asyncio.Lock] = {}
+        self._setup_claimed = False
         self.stopped = False
         """Whether teardown has begun (set by `stop`). A stopped runtime is dead: a rollout
         refuses to borrow one — the owner tore it down, so any use is a lifetime bug in the
@@ -244,15 +273,26 @@ class Runtime(ABC):
 
     async def prepare_setup(self) -> None:
         """Claim the runtime for trusted setup; restricted runtimes may reject reuse."""
+        if not self.network_restricted:
+            return
+        if self._setup_claimed:
+            raise SandboxError(
+                f"network-filtered {self.type} runtimes are single-rollout; "
+                "provision a fresh runtime instead of reusing this one"
+            )
+        self._setup_claimed = True
 
     async def prepare_execution(self, routes: list[str]) -> None:
         """Last setup step, right before the agent starts. Restricted runtimes enforce
         their policy here while keeping the interception and MCP `routes` reachable."""
 
     @property
-    def network_isolated(self) -> bool:
+    def network_restricted(self) -> bool:
         """Whether the agent phase uses filtered networking (see `prepare_execution`)."""
-        return False
+        return (
+            isinstance(self.config, NetworkPolicyConfig)
+            and self.config.network_restricted
+        )
 
     @property
     def published_port(self) -> int | None:

{verifiers-0.2.2.dev22 → verifiers-0.2.2.dev24}/verifiers/v1/runtimes/docker/__init__.py

@@ -13,11 +13,10 @@ from pathlib import PurePosixPath
 from typing import Literal
 from urllib.parse import urlsplit
 
-from pydantic_config import BaseConfig
-
 from verifiers.v1.errors import SandboxError
 from verifiers.v1.runtimes.base import (
     BaseRuntimeInfo,
+    NetworkPolicyConfig,
     ProgramResult,
     Runtime,
     parse_gpu,
@@ -27,7 +26,7 @@ from verifiers.v1.runtimes.docker.egress import HOST_ALIAS, EgressProxy, Network
 logger = logging.getLogger(__name__)
 
 
-class DockerConfig(BaseConfig):
+class DockerConfig(NetworkPolicyConfig):
     type: Literal["docker"] = "docker"
     image: str = "python:3.11-slim"
     workdir: str = "/app"
@@ -42,17 +41,6 @@ class DockerConfig(BaseConfig):
     disk: float | None = None
     """Advisory disk request in GB. Docker has no portable per-container size limit, so
     this is accepted (so a task can declare it without a warning) but not enforced."""
-    allow: list[str] = []
-    """URL origins or host patterns the agent may reach during execution. Framework
-    routes are always added; an empty list allows only those routes and `*` allows all."""
-    block: list[str] = []
-    """URL origins or host patterns denied during execution. Block rules win over
-    `allow`; framework interception and MCP routes always remain reachable."""
-
-    @property
-    def network_isolated(self) -> bool:
-        """True unless a bare wildcard permits every destination without exceptions."""
-        return "*" not in self.allow or bool(self.block)
 
 
 class DockerRuntimeInfo(DockerConfig, BaseRuntimeInfo):
@@ -96,7 +84,6 @@ class DockerRuntime(Runtime):
         self._proxy: EgressProxy | None = None
         self._proxy_host_ip: str | None = None
         self._stopped = False
-        self._setup_claimed = False
         self._cut = False
 
     async def start(self) -> None:
@@ -127,8 +114,8 @@ class DockerRuntime(Runtime):
         _, gpu_count = parse_gpu(self.config.gpu)
         if gpu_count:
             limits += ["--gpus", str(gpu_count)]
-        isolated = self.network_isolated
-        if isolated:
+        restricted = self.network_restricted
+        if restricted:
             network = [
                 "--network",
                 "bridge",
@@ -164,7 +151,7 @@ class DockerRuntime(Runtime):
         self.info.id = run.stdout.strip()[
             :12
         ]  # `docker run -d` prints the container id
-        if isolated:
+        if restricted:
             # Setup is trusted; colocated servers fetch their task from host interception
             # before the final framework routes are known.
             self._proxy = EgressProxy(
@@ -236,33 +223,19 @@ class DockerRuntime(Runtime):
         host = urlsplit(url).hostname
         # Keep numeric loopback container-local; the proxy maps this reserved name to
         # the Verifiers process's loopback for interception and host-local MCP routes.
-        if self.network_isolated and host in ("127.0.0.1", "localhost"):
+        if self.network_restricted and host in ("127.0.0.1", "localhost"):
             return url.replace(host, HOST_ALIAS, 1)
         if (
-            not self.network_isolated
+            not self.network_restricted
             and sys.platform != "linux"
             and host in ("127.0.0.1", "localhost")
         ):
             return url.replace(host, "host.docker.internal", 1)
         return url
 
-    @property
-    def network_isolated(self) -> bool:
-        return self.config.network_isolated
-
-    async def prepare_setup(self) -> None:
-        if not self.network_isolated:
-            return
-        if self._setup_claimed:
-            raise SandboxError(
-                "network-filtered Docker runtimes are single-rollout; provision a "
-                "fresh runtime instead of reusing this one"
-            )
-        self._setup_claimed = True
-
     async def prepare_execution(self, routes: list[str]) -> None:
         """Allow the declared framework routes, then leave the proxy as the only route."""
-        if not self.network_isolated:
+        if not self.network_restricted:
             return
         assert self._proxy is not None
         framework = [

{verifiers-0.2.2.dev22 → verifiers-0.2.2.dev24}/verifiers/v1/runtimes/prime.py

@@ -14,14 +14,15 @@ import shlex
 import tempfile
 from pathlib import Path, PurePosixPath
 from typing import ClassVar, Literal
+from urllib.parse import urlsplit
 
 from pydantic import model_validator
-from pydantic_config import BaseConfig
 
 from verifiers.v1.errors import SandboxError
 from verifiers.v1.runtimes.base import (
     SERVICE_PORT,
     BaseRuntimeInfo,
+    NetworkPolicyConfig,
     ProgramResult,
     Runtime,
     parse_gpu,
@@ -34,7 +35,7 @@ MAX_LIFETIME = 24 * 60 * 60
 """Prime's fixed cap (seconds) on any sandbox's total lifetime."""
 
 
-class PrimeConfig(BaseConfig):
+class PrimeConfig(NetworkPolicyConfig):
     type: Literal["prime"] = "prime"
     image: str = "python:3.11-slim"
     """Docker image to run. Any pullable ref works: on the first use of an image, the
@@ -42,7 +43,6 @@ class PrimeConfig(BaseConfig):
     ~10 minutes) and caches the result, so later sandboxes on the same ref start in
     seconds."""
     workdir: str = "/app"
-    network_access: bool = True
     vm: bool = False
     """Run as a micro-VM rather than a container (kernel features / stronger isolation)."""
     guaranteed: bool = False
@@ -67,6 +67,22 @@ class PrimeConfig(BaseConfig):
     env-server worker process (None/<= 0 disables it). (Tunnel creation is limited separately
     and globally — see interception.tunnel.prime.TUNNEL_LIMITER.)"""
 
+    @model_validator(mode="after")
+    def _validate_egress(self) -> "PrimeConfig":
+        if not self.network_restricted:
+            return self
+        if not self.vm:
+            raise ValueError(
+                "Prime allow/block egress lists require a VM sandbox (vm=true)"
+            )
+        from prime_sandboxes.models import validate_egress_lists
+
+        allow = None if self.allow == ["*"] else self.allow
+        block = self.block or None
+        if allow is not None or block != ["*"]:
+            validate_egress_lists(allow, block)
+        return self
+
     @model_validator(mode="after")
     def _validate_idle_timeout(self) -> "PrimeConfig":
         if self.idle_timeout is not None and self.idle_timeout > MAX_LIFETIME:
@@ -133,7 +149,6 @@ class PrimeRuntime(Runtime):
                         name=self.name,
                         labels=self.config.labels,
                         docker_image=self.config.image,
-                        network_access=self.config.network_access,
                         vm=self.config.vm,
                         guaranteed=self.config.guaranteed,
                         **{k: v for k, v in options.items() if v is not None},
@@ -165,6 +180,44 @@ class PrimeRuntime(Runtime):
         ) as e:  # provisioning failure is one rollout's problem, not the eval's
             raise SandboxError(f"prime sandbox provisioning failed: {e}") from e
 
+    async def prepare_execution(self, routes: list[str]) -> None:
+        """Apply the host policy after setup and wait until the platform enforces it."""
+        if not self.network_restricted:
+            return
+        try:
+            if self.config.allow == ["*"]:
+                policy = {"deny": self.config.block}
+            else:
+                hosts = [h for h in (urlsplit(route).hostname for route in routes) if h]
+                entries = list(dict.fromkeys([*hosts, *self.config.allow]))
+                from prime_sandboxes.models import validate_egress_lists
+
+                validate_egress_lists(entries, None)
+                policy = {"allow": entries} if entries else {"deny": ["*"]}
+            status = await self._client.set_network(self.info.id, **policy)
+            try:
+                async with asyncio.timeout(60):
+                    delay = 0.1
+                    while not status.applied:
+                        await asyncio.sleep(delay)
+                        delay = min(delay * 2, 3)
+                        status = await self._client.get_network(self.info.id)
+            except TimeoutError as e:
+                raise SandboxError(
+                    "prime egress policy was not applied within 60s on sandbox "
+                    f"{self.info.id}; refusing to start the agent unrestricted"
+                ) from e
+        except SandboxError:
+            raise
+        except Exception as e:
+            raise SandboxError(f"prime egress policy failed: {e}") from e
+        logger.info(
+            "prime: egress policy applied on sandbox %s (allow=%s block=%s)",
+            self.info.id,
+            self.config.allow,
+            self.config.block,
+        )
+
     async def run(self, argv: list[str], env: dict[str, str]) -> ProgramResult:
         try:
             # Poll directly so the rollout stage owns the execution timeout; the SDK helper

{verifiers-0.2.2.dev22 → verifiers-0.2.2.dev24}/verifiers/v1/task.py

@@ -140,11 +140,11 @@ class TaskData(StrictBaseModel):
     workdir: str | None = None
     network_allow: list[str] = ["*"]
     """Execution-time destinations requested by this task. `*` leaves the runtime
-    allowlist unchanged; a concrete list replaces a wildcard or combines with
-    existing entries."""
+    allowlist unchanged; a concrete list replaces a wildcard or combines with existing
+    entries. Prime runtimes accept host-level entries and require `vm=true`."""
     network_block: list[str] = []
     """Execution-time destinations denied by this task and combined with runtime
-    blocks."""
+    blocks. Prime runtimes cannot combine an allowlist and a blocklist."""
     timeout: TaskTimeout = TaskTimeout()
     resources: TaskResources = TaskResources()
 

{verifiers-0.2.2.dev22 → verifiers-0.2.2.dev24}/verifiers/v1/utils/compile.py

@@ -7,7 +7,7 @@ from collections.abc import Collection
 
 from verifiers.v1.harness import Harness
 from verifiers.v1.runtimes import (
-    DockerConfig,
+    NetworkPolicyConfig,
     RuntimeConfig,
     SubprocessConfig,
     runtime_is_local,
@@ -45,19 +45,13 @@ def resolve_runtime_config(
         task.data.network_block
     )
     if task_network_policy:
-        if not isinstance(config, DockerConfig):
+        if not isinstance(config, NetworkPolicyConfig):
             raise ValueError(
-                f"task {task.data.idx!r} requires a URL network policy, but the "
-                f"{config.type} runtime does not support framework-aware URL policies"
+                f"task {task.data.idx!r} requires a network policy, but the "
+                f"{config.type} runtime does not support framework-aware policies"
             )
-        if "*" not in task.data.network_allow:
-            updates["allow"] = (
-                task.data.network_allow
-                if "*" in config.allow
-                else list(dict.fromkeys([*task.data.network_allow, *config.allow]))
-            )
-        updates["block"] = list(
-            dict.fromkeys([*task.data.network_block, *config.block])
+        config = config.with_task_network_policy(
+            task.data.network_allow, task.data.network_block
         )
     for resource, value in task.data.resources.model_dump(exclude_none=True).items():
         spec = type(config).model_fields.get(resource)