verifiers 0.3.2.dev138__tar.gz → 0.3.2.dev139__tar.gz

{verifiers-0.3.2.dev138 → verifiers-0.3.2.dev139}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.3.2.dev138
+Version: 0.3.2.dev139
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.3.2.dev138 → verifiers-0.3.2.dev139}/verifiers/v1/runtimes/base.py

@@ -72,6 +72,10 @@ class RuntimeProcess(ABC):
     async def wait(self) -> int:
         pass
 
+    @abstractmethod
+    async def poll(self) -> int | None:
+        """Return the underlying process's exit code without waiting for it to exit."""
+
     @abstractmethod
     async def terminate(self) -> None:
         pass
@@ -134,7 +138,7 @@ class BaseRuntimeInfo(BaseConfig):
 class Runtime(ABC):
     __slots__ = ("env",)
 
-    is_local: ClassVar[bool] = True
+    is_local: bool = True
     """Whether this runtime exchanges host-local URLs without a public tunnel. True for
     subprocess and the local container runtimes; remote runtimes override to False and
     use a host `Tunnel` inward plus `expose` outward."""

{verifiers-0.3.2.dev138 → verifiers-0.3.2.dev139}/verifiers/v1/runtimes/container.py

@@ -1,4 +1,4 @@
-"""Shared machinery for local container runtimes driven through a CLI `exec`."""
+"""Container operations through a CLI on the local machine or an owned runtime."""
 
 import asyncio
 import contextlib
@@ -7,14 +7,19 @@ import shlex
 import signal
 import uuid
 from pathlib import PurePosixPath
+from typing import TYPE_CHECKING
 
 from pydantic_config import BaseConfig
 
 from verifiers.v1.errors import SandboxError
 from verifiers.v1.runtimes.base import ProgramResult, Runtime, RuntimeProcess
-from verifiers.v1.runtimes.subprocess import read_stream
+from verifiers.v1.runtimes.subprocess import SubprocessProcess
 from verifiers.v1.utils.aio import run_shielded
 
+if TYPE_CHECKING:
+    from verifiers.v1.runtimes.modal import ModalConfig
+    from verifiers.v1.runtimes.prime import PrimeConfig
+
 
 class ContainerConfig(BaseConfig):
     image: str = "python:3.11-slim"
@@ -37,11 +42,12 @@ class ContainerConfig(BaseConfig):
 
 
 async def _communicate(
-    *argv: str, input: bytes | None = None
+    *argv: str, input: bytes | None = None, env: dict[str, str] | None = None
 ) -> tuple[int, bytes, bytes]:
-    """Run a host command to completion; a cancelled await kills it first."""
+    """Run a host command; explicit env replaces inheritance, cancellation kills it."""
     proc = await asyncio.create_subprocess_exec(
         *argv,
+        env=env,
         stdin=asyncio.subprocess.PIPE
         if input is not None
         else asyncio.subprocess.DEVNULL,
@@ -59,8 +65,10 @@ async def _communicate(
     return proc.returncode or 0, stdout, stderr
 
 
-async def cli(*argv: str, input: bytes | None = None) -> ProgramResult:
-    code, stdout, stderr = await _communicate(*argv, input=input)
+async def cli(
+    *argv: str, input: bytes | None = None, env: dict[str, str] | None = None
+) -> ProgramResult:
+    code, stdout, stderr = await _communicate(*argv, input=input, env=env)
     return ProgramResult(
         code, stdout.decode(errors="replace"), stderr.decode(errors="replace")
     )
@@ -68,35 +76,38 @@ async def cli(*argv: str, input: bytes | None = None) -> ProgramResult:
 
 class ContainerProcess(RuntimeProcess):
     """A process attached through the CLI client. The client does not forward signals,
-    so `exec` (the argv prefix that runs a command inside the container) delivers them."""
+    so the container runtime delivers them directly to the inner process group."""
 
     def __init__(
         self,
-        process: asyncio.subprocess.Process,
-        exec: list[str],
+        process: RuntimeProcess,
+        runtime: "ContainerRuntime",
         pid: int,
         pidfile: str,
     ) -> None:
         self._process = process
-        self._exec = exec
+        self._runtime = runtime
         self._pid = pid
         self._pidfile = pidfile
-        assert process.stdin is not None
-        assert process.stdout is not None
-        assert process.stderr is not None
-        self._stdin = process.stdin
-        self.stdout = read_stream(process.stdout)
-        self.stderr = read_stream(process.stderr)
+        self.stdout, self.stderr = process.stdout, process.stderr
 
     async def write(self, data: bytes) -> None:
-        self._stdin.write(data)
-        await self._stdin.drain()
+        await self._process.write(data)
 
     async def wait(self) -> int:
         try:
             return await self._process.wait()
         finally:
-            await run_shielded(cli(*self._exec, "rm", "-f", self._pidfile))
+            # Removing bookkeeping files must not replace the process result.
+            with contextlib.suppress(Exception):
+                await run_shielded(
+                    self._runtime._run_host(
+                        *self._runtime._exec({}), "rm", "-f", self._pidfile
+                    )
+                )
+
+    async def poll(self) -> int | None:
+        return await self._process.poll()
 
     async def terminate(self) -> None:
         await self._signal("TERM")
@@ -105,27 +116,27 @@ class ContainerProcess(RuntimeProcess):
         await self._signal("KILL")
 
     async def _signal(self, signal: str) -> None:
-        if self._process.returncode is not None:
+        if await self.poll() is not None:
             return
-        result = await cli(
-            *self._exec,
+        result = await self._runtime._run_host(
+            *self._runtime._exec({}),
             "sh",
             "-c",
-            'kill -"$1" "-$2" 2>/dev/null || kill -"$1" "$2"',
+            'kill -"$1" "-$2" 2>/dev/null || kill -"$1" "$2" 2>/dev/null',
             "vf-signal",
             signal,
             str(self._pid),
         )
-        if result.exit_code != 0 and self._process.returncode is None:
+        if result.exit_code != 0 and await self.poll() is None:
             raise SandboxError(
                 f"container process signal failed: {result.stderr.strip()}"
             )
 
 
 async def _abort_process_startup(
-    proc: asyncio.subprocess.Process, exec: list[str], pidfile: str
+    proc: RuntimeProcess, runtime: "ContainerRuntime", pidfile: str
 ) -> str:
-    """Kill a partially opened container process and reap its local CLI client."""
+    """Kill a partially opened container process and reap its CLI client."""
     # The target normally writes its PID immediately, but cancellation can win
     # that race. Wait briefly for the file before signalling the process group.
     cleanup = (
@@ -137,23 +148,64 @@ async def _abort_process_startup(
     )
     try:
         with contextlib.suppress(Exception):
+            # An exited CLI client may have left its inner process running.
             await asyncio.wait_for(
-                cli(*exec, "sh", "-c", cleanup, "vf-process-cleanup", pidfile),
-                timeout=2,
+                runtime._run_host(
+                    *runtime._exec({}),
+                    "sh",
+                    "-c",
+                    cleanup,
+                    "vf-process-cleanup",
+                    pidfile,
+                ),
+                timeout=5,
             )
     finally:
-        if proc.returncode is None:
-            with contextlib.suppress(ProcessLookupError):
-                proc.kill()
-        _, stderr = await proc.communicate()
+        with contextlib.suppress(Exception):
+            await asyncio.wait_for(proc.kill(), 5)
+        stderr = b""
+        with contextlib.suppress(Exception):
+            async with asyncio.timeout(5):
+                async for _ in proc.stdout:
+                    pass
+                stderr = b"".join([chunk async for chunk in proc.stderr])
+                await proc.wait()
     return stderr.decode(errors="replace").strip()
 
 
 class ContainerRuntime(Runtime):
-    """A local container reached through its CLI: every operation is an `exec` into it.
+    """A container reached through its CLI: every operation is an `exec` into it.
     Subclasses provision the container (`start` / `cleanup`) and describe the exec."""
 
-    config: ContainerConfig
+    config: "ContainerConfig | PrimeConfig | ModalConfig"
+    _host: Runtime | None = None
+
+    async def _run_host(self, *argv: str) -> ProgramResult:
+        if self._host is None:
+            return await cli(*argv)
+        return await self._host.run(list(argv), {})
+
+    async def _communicate_host(
+        self, *argv: str, input: bytes | None = None
+    ) -> tuple[int, bytes, bytes]:
+        if self._host is None:
+            return await _communicate(*argv, input=input)
+        # Stage bytes through the provider filesystem, never its text command logs.
+        temporary = f"/tmp/vf-io-{uuid.uuid4().hex}"
+        command = f"{shlex.join(argv)} > {temporary}.out"
+        try:
+            if input is not None:
+                await self._host.write(f"{temporary}.in", input)
+                command += f" < {temporary}.in"
+            result = await self._run_host("sh", "-c", command)
+            data = await self._host.read(f"{temporary}.out")
+            return result.exit_code, data, result.stderr.encode()
+        finally:
+            # Temporary-file cleanup is best-effort after command completion/failure.
+            with contextlib.suppress(Exception):
+                await run_shielded(
+                    self._run_host("rm", "-f", f"{temporary}.in", f"{temporary}.out")
+                )
 
     def _exec(self, env: dict[str, str], *, stdin: bool = False) -> list[str]:
         """Host argv that runs a command inside the container, in the workdir, with
@@ -161,7 +213,7 @@ class ContainerRuntime(Runtime):
         raise NotImplementedError
 
     async def run(self, argv: list[str], env: dict[str, str]) -> ProgramResult:
-        return await cli(*self._exec(self.process_env(env)), *argv)
+        return await self._run_host(*self._exec(self.process_env(env)), *argv)
 
     async def open_process(
         self, argv: list[str], env: dict[str, str]
@@ -177,8 +229,7 @@ class ContainerRuntime(Runtime):
             'vf-process "$@"; '
             'fi; echo $$ > "$1"; shift; exec "$@"'
         )
-        control = self._exec({})
-        proc = await asyncio.create_subprocess_exec(
+        command = [
             *self._exec(self.process_env(env), stdin=True),
             "sh",
             "-c",
@@ -186,33 +237,47 @@ class ContainerRuntime(Runtime):
             "vf-process",
             pidfile,
             *argv,
-            stdin=asyncio.subprocess.PIPE,
-            stdout=asyncio.subprocess.PIPE,
-            stderr=asyncio.subprocess.PIPE,
-        )
+        ]
+        if self._host is None:
+            proc = SubprocessProcess(
+                await asyncio.create_subprocess_exec(
+                    *command,
+                    stdin=asyncio.subprocess.PIPE,
+                    stdout=asyncio.subprocess.PIPE,
+                    stderr=asyncio.subprocess.PIPE,
+                    start_new_session=True,
+                )
+            )
+        else:
+            proc = await self._host.open_process(command, {})
         loop = asyncio.get_running_loop()
-        deadline = loop.time() + 5
-        exited = False
+        deadline = loop.time() + (30 if self._host is not None else 5)
         try:
-            while True:
-                ready = await cli(*control, "cat", pidfile)
-                if ready.exit_code == 0 and ready.stdout.strip().isdigit():
-                    return ContainerProcess(
-                        proc, control, int(ready.stdout.strip()), pidfile
-                    )
-                # A target that already exited still left its pidfile: poll once more.
-                if exited or loop.time() >= deadline:
-                    break
-                exited = proc.returncode is not None
-                await asyncio.sleep(0.05)
+            async with asyncio.timeout_at(deadline):
+                while True:
+                    # An exited client gets one final PID read: short commands may
+                    # have written their PID and finished before the first poll.
+                    returncode = await proc.poll()
+                    ready = await self._run_host(*self._exec({}), "cat", pidfile)
+                    if ready.exit_code == 0 and ready.stdout.strip().isdigit():
+                        return ContainerProcess(
+                            proc, self, int(ready.stdout.strip()), pidfile
+                        )
+                    if returncode is not None:
+                        break
+                    await asyncio.sleep(0.05)
+        except TimeoutError as error:
+            stderr = await run_shielded(_abort_process_startup(proc, self, pidfile))
+            raise SandboxError(
+                f"container live process failed to start: {stderr or 'PID unavailable'}"
+            ) from error
         except BaseException:
-            await run_shielded(_abort_process_startup(proc, control, pidfile))
+            await run_shielded(_abort_process_startup(proc, self, pidfile))
             raise
-
-        stderr = await run_shielded(_abort_process_startup(proc, control, pidfile))
-        detail = stderr or ready.stderr.strip()
+        stderr = await run_shielded(_abort_process_startup(proc, self, pidfile))
         raise SandboxError(
-            f"container live process failed to start: {detail or 'PID unavailable'}"
+            f"container client exited with code {returncode} before reporting its PID: "
+            f"{stderr or ready.stderr.strip()}"
         )
 
     async def run_background(
@@ -221,7 +286,9 @@ class ContainerRuntime(Runtime):
         # Backgrounded inside the container, so it outlives this exec and lives until
         # the container is removed in stop().
         script = f"{shlex.join(argv)} > {shlex.quote(log)} 2>&1 < /dev/null &"
-        result = await cli(*self._exec(self.process_env(env)), "sh", "-c", script)
+        result = await self._run_host(
+            *self._exec(self.process_env(env)), "sh", "-c", script
+        )
         if result.exit_code != 0:
             raise SandboxError(
                 f"container background process failed: {result.stderr.strip()}"
@@ -229,7 +296,9 @@ class ContainerRuntime(Runtime):
 
     async def _read(self, path: str, max_bytes: int | None = None) -> bytes:
         argv = ["cat"] if max_bytes is None else ["head", "-c", str(max_bytes)]
-        code, data, stderr = await _communicate(*self._exec({}), *argv, "--", path)
+        code, data, stderr = await self._communicate_host(
+            *self._exec({}), *argv, "--", path
+        )
         if code != 0:
             raise SandboxError(
                 f"read {path!r}: {stderr.decode(errors='replace').strip()}"
@@ -238,12 +307,14 @@ class ContainerRuntime(Runtime):
 
     async def write(self, path: str, data: bytes) -> None:
         parent = shlex.quote(str(PurePosixPath(path).parent))
-        result = await cli(
+        code, _, stderr = await self._communicate_host(
             *self._exec({}, stdin=True),
             "sh",
             "-c",
             f"mkdir -p {parent} && cat > {shlex.quote(path)}",
             input=data,
         )
-        if result.exit_code != 0:
-            raise SandboxError(f"write {path!r}: {result.stderr.strip()}")
+        if code != 0:
+            raise SandboxError(
+                f"write {path!r}: {stderr.decode(errors='replace').strip()}"
+            )

{verifiers-0.3.2.dev138 → verifiers-0.3.2.dev139}/verifiers/v1/runtimes/docker/__init__.py

@@ -12,7 +12,8 @@ import subprocess
 import sys
 import tempfile
 import uuid
-from typing import ClassVar, Literal
+from collections.abc import AsyncIterator
+from typing import TYPE_CHECKING, ClassVar, Literal
 from urllib.parse import urlsplit
 
 from verifiers.v1.configs.runtime import NetworkPolicyConfig
@@ -20,6 +21,7 @@ from verifiers.v1.errors import SandboxError
 from verifiers.v1.runtimes.base import (
     SERVICE_PORT,
     BaseRuntimeInfo,
+    Runtime,
     parse_gpu,
 )
 from verifiers.v1.runtimes.container import ContainerConfig, ContainerRuntime, cli
@@ -32,6 +34,10 @@ from verifiers.v1.utils.scope import run_scope
 
 logger = logging.getLogger(__name__)
 
+if TYPE_CHECKING:
+    from verifiers.v1.runtimes.modal import ModalConfig
+    from verifiers.v1.runtimes.prime import PrimeConfig
+
 
 class DockerConfig(ContainerConfig, NetworkPolicyConfig):
     type: Literal["docker"] = "docker"
@@ -69,17 +75,77 @@ class DockerRuntime(ContainerRuntime):
     info_cls: ClassVar[type[BaseRuntimeInfo]] = DockerRuntimeInfo
 
     def __init__(
-        self, config: DockerConfig | PodmanConfig, name: str | None = None
+        self,
+        config: "DockerConfig | PodmanConfig | PrimeConfig | ModalConfig",
+        name: str | None = None,
+        *,
+        host: Runtime | None = None,
     ) -> None:
         super().__init__(name)
         self.config = config.model_copy(update={"workdir": config.workdir or "/app"})
-        self.info = self.info_cls(**self.config.model_dump())
+        self._host = host
+        self.is_local = host is None or host.is_local
+        self.info = (
+            host.info.model_copy(
+                update={"image": self.config.image, "workdir": self.config.workdir}
+            )
+            if host
+            else self.info_cls(**self.config.model_dump())
+        )
         self._container: str | None = None  # our `--name` (used for exec/rm)
         self._service_url: str | None = None
         self._proxy: EgressProxy | None = None
         self._image_env: dict[str, str] = {}
         self._stopped = False
         self._cut = False
+        # Helper containers carry the run label, including for borrowed containers.
+        self._label_args = ["--label", f"verifiers.run={run_scope()}"]
+
+    @classmethod
+    @contextlib.asynccontextmanager
+    async def attach(
+        cls,
+        config: "DockerConfig | PrimeConfig | ModalConfig",
+        container: str,
+        *,
+        service_url: str | None = None,
+        host: Runtime | None = None,
+    ) -> AsyncIterator["DockerRuntime"]:
+        """Borrow an existing container; its caller owns creation and removal."""
+        runtime = cls(config, host=host)
+        runtime._container = container
+        runtime.info.borrowed = True  # cleanup never removes a borrowed container
+        try:
+            code, data, stderr = await runtime._communicate_host(
+                runtime.engine, "inspect", "--format", "{{json .Config}}", container
+            )
+            if code:
+                raise SandboxError(
+                    f"Container inspection failed: {stderr.decode(errors='replace')}"
+                )
+            info = json.loads(data)
+            runtime._image_env = dict(
+                entry.split("=", 1) for entry in info["Env"] or []
+            )
+            runtime.config = config.model_copy(
+                update={"image": info["Image"], "workdir": info["WorkingDir"] or "/"}
+            )
+            if host is None:
+                runtime.info.id = container
+            runtime.info.image = runtime.config.image
+            runtime.info.workdir = runtime.config.workdir
+            runtime._service_url = service_url
+            if host is None and service_url is not None:
+                runtime._proxy = EgressProxy(NetworkPolicy(NetworkPolicyConfig(), []))
+                if sys.platform == "linux":
+                    await runtime._proxy.start(
+                        listener=await runtime._container_listener()
+                    )
+                else:
+                    await runtime._proxy.start("127.0.0.1")
+            yield runtime
+        finally:
+            await runtime.stop()
 
     @property
     def published_port(self) -> int:
@@ -162,7 +228,6 @@ class DockerRuntime(ContainerRuntime):
             for key, value in self.env.items()
             for arg in ("--env", f"{key}={value}")
         ]
-        self._label_args = ["--label", f"verifiers.run={run_scope()}"]
         run = await cli(
             self.engine,
             "run",
@@ -262,6 +327,8 @@ class DockerRuntime(ContainerRuntime):
         )
 
     def host_url(self, url: str) -> str:
+        if self._host is not None:
+            return self._host.host_url(url)
         parts = urlsplit(url)
         if not is_loopback_host(parts.hostname or ""):
             return url
@@ -270,6 +337,8 @@ class DockerRuntime(ContainerRuntime):
         return self._proxy.callback_url(url, host)
 
     async def expose(self, port: int) -> str:
+        if self._host is not None:
+            return await self._host.expose(port)
         if port != SERVICE_PORT or self._service_url is None:
             raise SandboxError(
                 f"{self.engine} publishes only port {SERVICE_PORT}, not {port}"
@@ -340,6 +409,9 @@ class DockerRuntime(ContainerRuntime):
 
     async def prepare_execution(self, routes: list[str] | None) -> None:
         """Allow the declared framework routes, then leave the proxy as the only way out."""
+        if self._host is not None:
+            await self._host.prepare_execution(routes)
+            return
         if not self.network_restricted:
             return
         assert self._proxy is not None
@@ -352,6 +424,7 @@ class DockerRuntime(ContainerRuntime):
             urlsplit(url)._replace(path="", query="", fragment="").geturl()
             for url in routes
         ]
+        assert isinstance(self.config, NetworkPolicyConfig)
         self._proxy.policy = NetworkPolicy(self.config, framework)
         if self._cut:
             return
@@ -465,20 +538,20 @@ class DockerRuntime(ContainerRuntime):
         self, argv: list[str], env: dict[str, str], log: str
     ) -> None:
         # A setup server outlives the network cut and needs the initially open proxy.
-        if self.network_restricted:
+        if self.network_restricted and self._proxy is not None:
             env = {**env, **self._proxy_env()}
         # The engine owns the background server as a detached exec process.
         command = self._exec(self.process_env(env))
         command.insert(2, "--detach")
         script = f"exec {shlex.join(argv)} > {shlex.quote(log)} 2>&1 < /dev/null"
-        result = await cli(*command, "sh", "-c", script)
+        result = await self._run_host(*command, "sh", "-c", script)
         if result.exit_code != 0:
             raise SandboxError(
                 f"{self.engine} background process failed: {result.stderr.strip()}"
             )
 
     def cleanup(self) -> None:
-        if self._container is None or self._stopped:
+        if self.info.borrowed or self._container is None or self._stopped:
             return
         self._stopped = (
             True  # idempotency guard; keep `_container` so the name still shows

{verifiers-0.3.2.dev138 → verifiers-0.3.2.dev139}/verifiers/v1/runtimes/modal.py

@@ -75,6 +75,8 @@ def _egress_domain(rule: str, *, framework: bool = False) -> str | None:
 
 class ModalConfig(NetworkPolicyConfig):
     type: Literal["modal"] = "modal"
+    vm: bool = False
+    """Use a VM sandbox for workloads requiring a Docker daemon."""
     image: str = "python:3.11-slim"
     workdir: str | None = None
     """Working directory override; None uses the task's workdir, or /app."""
@@ -113,6 +115,12 @@ class ModalConfig(NetworkPolicyConfig):
             _egress_domain(rule)
         return self
 
+    @model_validator(mode="after")
+    def _validate_vm(self) -> "ModalConfig":
+        if self.vm and self.gpu:
+            raise ValueError("Modal VM sandboxes support CPU workloads only")
+        return self
+
 
 class ModalRuntimeInfo(ModalConfig, BaseRuntimeInfo):
     pass
@@ -134,6 +142,9 @@ class ModalProcess(RuntimeProcess):
     async def wait(self) -> int:
         return await self._process.wait.aio()
 
+    async def poll(self) -> int | None:
+        return await self._process.poll.aio()
+
     async def terminate(self) -> None:
         await self._signal("TERM")
 
@@ -233,6 +244,7 @@ class ModalRuntime(Runtime):
             ),
             timeout=24 * 60 * 60,  # Maximum lifetime of any sandbox.
             encrypted_ports=[SERVICE_PORT],
+            experimental_options={"vm_runtime": True} if self.config.vm else {},
         )
 
     async def prepare_execution(self, routes: list[str] | None) -> None:

{verifiers-0.3.2.dev138 → verifiers-0.3.2.dev139}/verifiers/v1/runtimes/prime.py

@@ -133,6 +133,9 @@ class PrimeProcess(RuntimeProcess):
     async def wait(self) -> int:
         return await self._process.wait()
 
+    async def poll(self) -> int | None:
+        return self._process.returncode
+
     async def terminate(self) -> None:
         await self._process.terminate()
 

{verifiers-0.3.2.dev138 → verifiers-0.3.2.dev139}/verifiers/v1/runtimes/subprocess.py

@@ -70,6 +70,9 @@ class SubprocessProcess(RuntimeProcess):
     async def wait(self) -> int:
         return await self._process.wait()
 
+    async def poll(self) -> int | None:
+        return self._process.returncode
+
     async def terminate(self) -> None:
         signal_process(self._process, signal.SIGTERM)