verifiers 0.1.15.dev439__tar.gz → 0.1.15.dev440__tar.gz

{verifiers-0.1.15.dev439 → verifiers-0.1.15.dev440}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.1.15.dev439
+Version: 0.1.15.dev440
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.1.15.dev439 → verifiers-0.1.15.dev440}/verifiers/v1/__init__.py

@@ -82,6 +82,7 @@ from verifiers.v1.runtimes import (
     ProgramResult,
     Runtime,
     RuntimeConfig,
+    RuntimeInfo,
     SubprocessConfig,
 )
 from verifiers.v1.state import State, StateT
@@ -191,6 +192,7 @@ __all__ = [
     "ModelContext",
     "Runtime",
     "RuntimeConfig",
+    "RuntimeInfo",
     "ProgramResult",
     "SubprocessConfig",
     "DockerConfig",

{verifiers-0.1.15.dev439 → verifiers-0.1.15.dev440}/verifiers/v1/rollout.py

@@ -149,6 +149,9 @@ class Rollout:
             self.runtime_config, name=trace.id
         )  # ref set first → always tearable-down; named after the rollout for traceability
         runtime = self.runtime
+        # The live info object, shared by reference: the trace carries the full runtime config
+        # from the start, and the resource `id` lands on it the moment start() provisions.
+        trace.runtime = runtime.info
         ctx = self.ctx
         stops = discover_decorated(self.taskset, "stop")
         logger.info(

{verifiers-0.1.15.dev439 → verifiers-0.1.15.dev440}/verifiers/v1/runtimes/__init__.py

@@ -13,22 +13,35 @@ from pydantic import Field
 
 from verifiers.v1.runtimes.base import (
     HOST,
+    BaseRuntimeInfo,
     ProgramResult,
     Runtime,
     host_endpoint,
     reachable_url,
     register,
 )
-from verifiers.v1.runtimes.docker import DockerConfig, DockerRuntime
-from verifiers.v1.runtimes.modal import ModalConfig, ModalRuntime
-from verifiers.v1.runtimes.prime import PrimeConfig, PrimeRuntime
-from verifiers.v1.runtimes.subprocess import SubprocessConfig, SubprocessRuntime
+from verifiers.v1.runtimes.docker import DockerConfig, DockerRuntime, DockerRuntimeInfo
+from verifiers.v1.runtimes.modal import ModalConfig, ModalRuntime, ModalRuntimeInfo
+from verifiers.v1.runtimes.prime import PrimeConfig, PrimeRuntime, PrimeRuntimeInfo
+from verifiers.v1.runtimes.subprocess import (
+    SubprocessConfig,
+    SubprocessRuntime,
+    SubprocessRuntimeInfo,
+)
 
 RuntimeConfig = Annotated[
     SubprocessConfig | DockerConfig | PrimeConfig | ModalConfig,
     Field(discriminator="type"),
 ]
 
+RuntimeInfo = Annotated[
+    SubprocessRuntimeInfo | DockerRuntimeInfo | PrimeRuntimeInfo | ModalRuntimeInfo,
+    Field(discriminator="type"),
+]
+"""What a rollout's runtime looked like, as trace data: each runtime's info type is its full
+config plus the provisioned resource's `id` (see `BaseRuntimeInfo`). The discriminated union
+is `Trace.runtime`'s type, so a dumped trace round-trips to the exact info class."""
+
 
 def _runtime_cls(config: RuntimeConfig) -> type[Runtime]:
     if isinstance(config, PrimeConfig):
@@ -58,6 +71,8 @@ __all__ = [
     "ProgramResult",
     "Runtime",
     "RuntimeConfig",
+    "RuntimeInfo",
+    "BaseRuntimeInfo",
     "make_runtime",
     "runtime_is_local",
     "host_endpoint",
@@ -65,10 +80,14 @@ __all__ = [
     "HOST",
     "SubprocessConfig",
     "SubprocessRuntime",
+    "SubprocessRuntimeInfo",
     "DockerConfig",
     "DockerRuntime",
+    "DockerRuntimeInfo",
     "PrimeConfig",
     "PrimeRuntime",
+    "PrimeRuntimeInfo",
     "ModalConfig",
     "ModalRuntime",
+    "ModalRuntimeInfo",
 ]

{verifiers-0.1.15.dev439 → verifiers-0.1.15.dev440}/verifiers/v1/runtimes/base.py

@@ -19,6 +19,8 @@ from dataclasses import dataclass
 from pathlib import PurePosixPath
 from typing import ClassVar, TypeVar
 
+from pydantic_config import BaseConfig
+
 from verifiers.v1.retries import retrying
 from verifiers.v1.utils.aio import run_shielded
 
@@ -106,6 +108,19 @@ def cleanup_at_exit() -> None:
             runtime.cleanup()
 
 
+class BaseRuntimeInfo(BaseConfig):
+    """Runtime metadata persisted with the rollout (`Trace.runtime`): each runtime's info class
+    inherits its full config — every knob the runtime ran with — and adds the provisioned
+    resource's `id`. Built with the runtime and shared by reference with the trace, so fields
+    land there the moment they're known: the config at creation, `id` once `start()` has
+    provisioned the resource. `RuntimeInfo` (the discriminated union over the concrete info
+    types) is what a dumped trace round-trips through."""
+
+    id: str | None = None
+    """The provisioned resource's identifier (None until the runtime is up): the subprocess
+    workdir, the docker container id, the prime/modal sandbox id."""
+
+
 class Runtime(ABC):
     is_local: ClassVar[bool] = True
     """Whether this runtime shares the host network — a program inside it reaches a host service
@@ -113,6 +128,11 @@ class Runtime(ABC):
     subprocess / docker(--network host); remote runtimes (modal/prime) override to False (they
     need a tunnel each way: `host_endpoint` inward, `expose` outward)."""
 
+    info: BaseRuntimeInfo
+    """This runtime's trace-facing metadata (see `BaseRuntimeInfo`): the full config plus the
+    resolved resource `id`. Each runtime constructs its own info type in `__init__` and fills
+    `id` in `start()`."""
+
     def __init__(self, name: str | None = None) -> None:
         self.name = name or f"vf-{uuid.uuid4().hex[:12]}"
         """Resource name — the subprocess workdir, docker `--name`, prime sandbox name.
@@ -131,9 +151,9 @@ class Runtime(ABC):
 
     @property
     def descriptor(self) -> str | None:
-        """A short resolved id for display (None until provisioned). Overridden per
-        runtime: subprocess workdir, docker image, prime sandbox id."""
-        return None
+        """A short resolved id for display (None until provisioned): the runtime's `info.id` —
+        subprocess workdir, docker container id, prime/modal sandbox id."""
+        return self.info.id
 
     # --- lifecycle ---
 

{verifiers-0.1.15.dev439 → verifiers-0.1.15.dev440}/verifiers/v1/runtimes/docker.py

@@ -15,7 +15,12 @@ from typing import Literal
 from pydantic_config import BaseConfig
 
 from verifiers.v1.errors import SandboxError
-from verifiers.v1.runtimes.base import ProgramResult, Runtime, parse_gpu
+from verifiers.v1.runtimes.base import (
+    BaseRuntimeInfo,
+    ProgramResult,
+    Runtime,
+    parse_gpu,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -37,6 +42,10 @@ class DockerConfig(BaseConfig):
     this is accepted (so a task can declare it without a warning) but not enforced."""
 
 
+class DockerRuntimeInfo(DockerConfig, BaseRuntimeInfo):
+    """`DockerConfig` + the resolved `id` — docker's short container id."""
+
+
 async def docker(*args: str) -> ProgramResult:
     """Run a `docker` CLI command, capturing its result."""
     proc = await asyncio.create_subprocess_exec(
@@ -59,14 +68,10 @@ class DockerRuntime(Runtime):
     def __init__(self, config: DockerConfig, name: str | None = None) -> None:
         super().__init__(name)
         self.config = config
+        self.info = DockerRuntimeInfo(**config.model_dump())
         self._container: str | None = None  # our `--name` (used for exec/rm)
-        self._container_id: str | None = None  # docker's short id (for display)
         self._stopped = False
 
-    @property
-    def descriptor(self) -> str | None:
-        return self._container_id
-
     async def start(self) -> None:
         try:
             version = await docker("version", "--format", "{{.Server.Version}}")
@@ -111,7 +116,7 @@ class DockerRuntime(Runtime):
         )
         if run.exit_code != 0:
             raise SandboxError(f"docker run failed: {run.stderr.strip()}")
-        self._container_id = run.stdout.strip()[
+        self.info.id = run.stdout.strip()[
             :12
         ]  # `docker run -d` prints the container id
         logger.info(

{verifiers-0.1.15.dev439 → verifiers-0.1.15.dev440}/verifiers/v1/runtimes/modal.py

@@ -17,7 +17,12 @@ from typing import ClassVar, Literal
 from pydantic_config import BaseConfig
 
 from verifiers.v1.errors import SandboxError
-from verifiers.v1.runtimes.base import SERVICE_PORT, ProgramResult, Runtime
+from verifiers.v1.runtimes.base import (
+    SERVICE_PORT,
+    BaseRuntimeInfo,
+    ProgramResult,
+    Runtime,
+)
 from verifiers.v1.runtimes.limiters import creation_limiter
 
 logger = logging.getLogger(__name__)
@@ -50,6 +55,10 @@ class ModalConfig(BaseConfig):
     env-server worker process (None/<= 0 disables it)."""
 
 
+class ModalRuntimeInfo(ModalConfig, BaseRuntimeInfo):
+    """`ModalConfig` + the resolved `id` — the modal sandbox object id."""
+
+
 class ModalRuntime(Runtime):
     """Runs the program in a Modal sandbox; the server is reached via a tunnel."""
 
@@ -58,12 +67,8 @@ class ModalRuntime(Runtime):
     def __init__(self, config: ModalConfig, name: str | None = None) -> None:
         super().__init__(name)
         self.config = config
+        self.info = ModalRuntimeInfo(**config.model_dump())
         self._sandbox = None
-        self._sandbox_id: str | None = None
-
-    @property
-    def descriptor(self) -> str | None:
-        return self._sandbox_id
 
     @property
     def published_port(self) -> int | None:
@@ -101,9 +106,9 @@ class ModalRuntime(Runtime):
                     timeout=24 * 60 * 60,  # Maximum lifetime of any sandbox.
                     encrypted_ports=[SERVICE_PORT],
                 )
-            self._sandbox_id = self._sandbox.object_id
+            self.info.id = self._sandbox.object_id
             logger.info(
-                "modal: sandbox %s up (image=%s)", self._sandbox_id, self.config.image
+                "modal: sandbox %s up (image=%s)", self.info.id, self.config.image
             )
             await self._sandbox.filesystem.make_directory.aio(self.config.workdir)
         except (
@@ -185,7 +190,7 @@ class ModalRuntime(Runtime):
         # the sandbox via Modal's sync API so the costly resource isn't left to its max-lifetime.
         # Idempotent — the async `stop` handles the normal path, and a second terminate is a no-op.
         sandbox, self._sandbox = self._sandbox, None
-        if sandbox is not None:  # `_sandbox_id` kept so descriptor survives teardown
+        if sandbox is not None:  # `info.id` kept so descriptor survives teardown
             with contextlib.suppress(Exception):
                 sandbox.terminate()
 
@@ -202,7 +207,5 @@ class ModalRuntime(Runtime):
         try:
             await sandbox.terminate.aio()
         except Exception as e:
-            logger.warning(
-                "modal: failed to terminate sandbox %s: %s", self._sandbox_id, e
-            )
+            logger.warning("modal: failed to terminate sandbox %s: %s", self.info.id, e)
         self._sandbox = None

{verifiers-0.1.15.dev439 → verifiers-0.1.15.dev440}/verifiers/v1/runtimes/prime.py

@@ -19,7 +19,13 @@ from pydantic import model_validator
 from pydantic_config import BaseConfig
 
 from verifiers.v1.errors import SandboxError
-from verifiers.v1.runtimes.base import SERVICE_PORT, ProgramResult, Runtime, parse_gpu
+from verifiers.v1.runtimes.base import (
+    SERVICE_PORT,
+    BaseRuntimeInfo,
+    ProgramResult,
+    Runtime,
+    parse_gpu,
+)
 from verifiers.v1.runtimes.limiters import creation_limiter
 
 logger = logging.getLogger(__name__)
@@ -69,6 +75,10 @@ class PrimeConfig(BaseConfig):
         return self
 
 
+class PrimeRuntimeInfo(PrimeConfig, BaseRuntimeInfo):
+    """`PrimeConfig` + the resolved `id` — the prime sandbox id."""
+
+
 class PrimeRuntime(Runtime):
     """Runs the program in a Prime sandbox; the server is reached via a tunnel."""
 
@@ -77,12 +87,8 @@ class PrimeRuntime(Runtime):
     def __init__(self, config: PrimeConfig, name: str | None = None) -> None:
         super().__init__(name)
         self.config = config
+        self.info = PrimeRuntimeInfo(**config.model_dump())
         self._client = None
-        self._sandbox_id: str | None = None
-
-    @property
-    def descriptor(self) -> str | None:
-        return self._sandbox_id
 
     @property
     def published_port(self) -> int | None:
@@ -96,10 +102,11 @@ class PrimeRuntime(Runtime):
         # GB). gpu_type/region are only sent when set (else provider-chosen).
         gpu_type, gpu_count = parse_gpu(self.config.gpu)
         # prime's idle timeout is in whole minutes; convert from the seconds config surface
-        # (floored to the SDK's 1-minute minimum).
+        # (floored to the SDK's 1-minute minimum). VM sandboxes don't support an idle timeout
+        # (the API 422s on it), so it's dropped there rather than failing every VM rollout.
         idle_minutes = (
             max(1, math.ceil(self.config.idle_timeout / 60))
-            if self.config.idle_timeout is not None
+            if self.config.idle_timeout is not None and not self.config.vm
             else None
         )
         options = {
@@ -130,13 +137,13 @@ class PrimeRuntime(Runtime):
                         **{k: v for k, v in options.items() if v is not None},
                     )
                 )
-            self._sandbox_id = sandbox.id
-            await self._client.wait_for_creation(self._sandbox_id)
+            self.info.id = sandbox.id
+            await self._client.wait_for_creation(self.info.id)
             logger.info(
-                "prime: sandbox %s up (image=%s)", self._sandbox_id, self.config.image
+                "prime: sandbox %s up (image=%s)", self.info.id, self.config.image
             )
             await self._client.run_background_job(
-                self._sandbox_id, f"mkdir -p {shlex.quote(self.config.workdir)}"
+                self.info.id, f"mkdir -p {shlex.quote(self.config.workdir)}"
             )
         except (
             Exception
@@ -148,14 +155,14 @@ class PrimeRuntime(Runtime):
             # Poll directly so the rollout stage owns the execution timeout; the SDK helper
             # otherwise imposes its own 15-minute limit.
             job = await self._client.start_background_job(
-                self._sandbox_id,
+                self.info.id,
                 shlex.join(argv),
                 working_dir=self.config.workdir,
                 env=env,
             )
             delay = 0.1
             while True:
-                result = await self._client.get_background_job(self._sandbox_id, job)
+                result = await self._client.get_background_job(self.info.id, job)
                 if result.completed:
                     break
                 await asyncio.sleep(delay)
@@ -177,7 +184,7 @@ class PrimeRuntime(Runtime):
         # backend default, which lands in us-central) 400 it; `us` supports it. TODO: re-enable the
         # prime cases in the e2e `skip_if_unexposable` guard once prime exposes ports in any region.
         try:
-            exposed = await self._client.expose(self._sandbox_id, port)
+            exposed = await self._client.expose(self.info.id, port)
         except Exception as e:  # surface prime's exposure constraints actionably
             raise SandboxError(
                 "prime port exposure failed — port exposure isn't supported in this sandbox's "
@@ -210,9 +217,7 @@ class PrimeRuntime(Runtime):
         try:
             with tempfile.TemporaryDirectory() as directory:
                 download = Path(directory) / "download"
-                await self._client.download_file(
-                    self._sandbox_id, target, str(download)
-                )
+                await self._client.download_file(self.info.id, target, str(download))
                 return await asyncio.to_thread(download.read_bytes)
         except Exception as e:
             raise SandboxError(f"read {path!r}: {e}") from e
@@ -234,7 +239,7 @@ class PrimeRuntime(Runtime):
         )
         try:
             await self._client.upload_bytes(
-                self._sandbox_id, target, data, filename=PurePosixPath(target).name
+                self.info.id, target, data, filename=PurePosixPath(target).name
             )
         except Exception as e:
             raise SandboxError(f"write {path!r}: {e}") from e
@@ -243,12 +248,12 @@ class PrimeRuntime(Runtime):
         # Synchronous atexit backstop (the async client can't run once the loop is gone): delete
         # the sandbox via the sync client, so the costly resource isn't left to its max-lifetime.
         # Idempotent — the async `stop` deletes it on the normal path, a second delete 404s.
-        if self._sandbox_id is not None:
+        if self.info.id is not None:
             from prime_sandboxes import SandboxClient
             from prime_sandboxes.core import APIClient
 
             with contextlib.suppress(Exception):
-                SandboxClient(APIClient()).delete(self._sandbox_id)
+                SandboxClient(APIClient()).delete(self.info.id)
 
     async def teardown(self) -> None:
         # Best-effort, idempotent teardown: delete the sandbox (the costly resource). Runs via
@@ -257,13 +262,13 @@ class PrimeRuntime(Runtime):
         if client is None:
             return
         if (
-            self._sandbox_id is not None
+            self.info.id is not None
         ):  # kept (not nulled) so descriptor survives teardown
             try:
-                await client.delete(self._sandbox_id)
+                await client.delete(self.info.id)
             except Exception as e:
                 logger.warning(
-                    "prime: failed to delete sandbox %s: %s", self._sandbox_id, e
+                    "prime: failed to delete sandbox %s: %s", self.info.id, e
                 )
         with contextlib.suppress(Exception):
             await client.aclose()

{verifiers-0.1.15.dev439 → verifiers-0.1.15.dev440}/verifiers/v1/runtimes/subprocess.py

@@ -15,7 +15,7 @@ from typing import Literal
 
 from pydantic_config import BaseConfig
 
-from verifiers.v1.runtimes.base import ProgramResult, Runtime
+from verifiers.v1.runtimes.base import BaseRuntimeInfo, ProgramResult, Runtime
 
 # A local subprocess inherits the host environment EXCEPT any var whose name
 # contains "API_KEY" — so it can never reach a real provider with the host's API
@@ -33,6 +33,10 @@ class SubprocessConfig(BaseConfig):
     type: Literal["subprocess"] = "subprocess"
 
 
+class SubprocessRuntimeInfo(SubprocessConfig, BaseRuntimeInfo):
+    """`SubprocessConfig` + the resolved `id` — the `/tmp/<name>` workspace path."""
+
+
 class SubprocessRuntime(Runtime):
     """Runs the program as a local subprocess in a unique /tmp workspace."""
 
@@ -45,16 +49,14 @@ class SubprocessRuntime(Runtime):
         self._uv_interpreters = self._interpreters
         self._uv_script_locks = self._locks
         self.config = config
+        self.info = SubprocessRuntimeInfo(**config.model_dump())
         self.workdir: Path | None = None
         self._background: list[asyncio.subprocess.Process] = []
 
-    @property
-    def descriptor(self) -> str | None:
-        return self.workdir.name if self.workdir else None
-
     async def start(self) -> None:
         self.workdir = Path("/tmp") / self.name
         self.workdir.mkdir()
+        self.info.id = str(self.workdir)
 
     async def run(self, argv: list[str], env: dict[str, str]) -> ProgramResult:
         full_env = {k: v for k, v in os.environ.items() if "API_KEY" not in k.upper()}

{verifiers-0.1.15.dev439 → verifiers-0.1.15.dev440}/verifiers/v1/trace.py

@@ -25,6 +25,7 @@ if TYPE_CHECKING:
 from verifiers.v1 import graph
 from verifiers.v1.errors import ProviderError
 from verifiers.v1.graph import MessageNode
+from verifiers.v1.runtimes import RuntimeInfo
 from verifiers.v1.state import State, StateT
 from verifiers.v1.task import TaskT, WireTask
 from verifiers.v1.types import (
@@ -220,6 +221,12 @@ class Trace(StrictBaseModel, Generic[TaskT, StateT]):
     """Unique id for this rollout, auto-generated per trace."""
     task: TaskT
     """The (immutable) task being solved — fully typed, flows into scoring."""
+    runtime: RuntimeInfo | None = None
+    """Where the rollout ran: the runtime's full config plus the provisioned resource's `id`
+    (subprocess workdir / docker container id / prime or modal sandbox id). The rollout assigns
+    the live runtime's `info` object the moment it makes the runtime, so the config is recorded
+    from the start and `id` appears as soon as the runtime is up. None only for traces created
+    outside a rollout."""
     nodes: list[MessageNode] = Field(default_factory=list)
     """The message graph — one node per distinct message, linked to its predecessor (see
     `graph`). The ground truth; `branches` is a view over it. Stores each message once, so