verifiers 0.2.2.dev74__tar.gz → 0.2.2.dev75__tar.gz

{verifiers-0.2.2.dev74 → verifiers-0.2.2.dev75}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.2.dev74
+Version: 0.2.2.dev75
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.2.2.dev74 → verifiers-0.2.2.dev75}/verifiers/v1/agent.py

@@ -29,7 +29,7 @@ from verifiers.v1.runtimes import (
     Runtime,
     RuntimeConfig,
     SubprocessConfig,
-    make_runtime,
+    provision_runtime,
     runtime_is_local,
 )
 from verifiers.v1.session import RolloutLimits
@@ -547,14 +547,8 @@ class Agent:
             if task is not None
             else self.runtime_config
         )
-        runtime = make_runtime(config)
-        try:
-            # start() inside the try: a failed start may already hold a remote
-            # sandbox, so it must reach stop() (safe on a partially-started runtime).
-            await runtime.start()
+        async with provision_runtime(config) as runtime:
             yield runtime
-        finally:
-            await runtime.stop()
 
 
 class _EpisodeAgent(Agent):

{verifiers-0.2.2.dev74 → verifiers-0.2.2.dev75}/verifiers/v1/runtimes/__init__.py

@@ -1,3 +1,5 @@
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager
 from typing import Annotated
 
 from pydantic import Field
@@ -45,6 +47,22 @@ def make_runtime(config: RuntimeConfig, name: str | None = None) -> Runtime:
     return runtime
 
 
+@asynccontextmanager
+async def provision_runtime(
+    config: RuntimeConfig, name: str | None = None
+) -> AsyncIterator[Runtime]:
+    """Provision a box from `config` and tear it down on exit.
+
+    `start()` sits inside the `try`: a failed start may already hold a paid sandbox, so
+    it has to reach `stop()` (which is safe on a partially-started runtime)."""
+    runtime = make_runtime(config, name)
+    try:
+        await runtime.start()
+        yield runtime
+    finally:
+        await runtime.stop()
+
+
 def runtime_is_local(config: RuntimeConfig) -> bool:
     """Whether a runtime of this config exchanges host-local URLs without a public
     tunnel, read off the runtime class without provisioning one."""
@@ -71,5 +89,6 @@ __all__ = [
     "SubprocessRuntime",
     "SubprocessRuntimeInfo",
     "make_runtime",
+    "provision_runtime",
     "runtime_is_local",
 ]

{verifiers-0.2.2.dev74 → verifiers-0.2.2.dev75}/verifiers/v1/runtimes/base.py

@@ -2,6 +2,7 @@
 
 import asyncio
 import atexit
+import base64
 import contextlib
 import hashlib
 import logging
@@ -285,9 +286,42 @@ class Runtime(ABC):
         argv = await self.prepare_uv_script(script, env)
         return await self.run([*argv, *(args or [])], env or {})
 
+    async def read(self, path: str, max_bytes: int | None = None) -> bytes:
+        """Read `path` into host memory. `max_bytes` caps the transfer, raising past
+        the cap — for a file written by something we don't control, whose size we
+        can't assume. The cap is enforced inside the box rather than after the
+        transfer, and base64 because `run` returns decoded text. Framework method —
+        override `_read`, not this."""
+        if max_bytes is None:
+            return await self._read(path)
+        # Through a temp file, not a pipe: `head | base64` exits with base64's 0
+        # even when the path is missing, and a missing file must raise here just
+        # as it does from `_read`.
+        result = await self.run(
+            [
+                "sh",
+                "-c",
+                (
+                    "t=$(mktemp) || exit 1; "
+                    'head -c "$1" -- "$2" > "$t" || { rm -f "$t"; exit 1; }; '
+                    'base64 < "$t"; rc=$?; rm -f "$t"; exit $rc'
+                ),
+                "sh",
+                str(max_bytes + 1),
+                path,
+            ],
+            {},
+        )
+        if result.exit_code:
+            raise SandboxError(f"read {path!r}: {result.stderr.strip()[-500:]}")
+        data = base64.b64decode(result.stdout)
+        if len(data) > max_bytes:
+            raise SandboxError(f"read {path!r}: over the {max_bytes} byte limit")
+        return data
+
     @abstractmethod
-    async def read(self, path: str) -> bytes:
-        pass
+    async def _read(self, path: str) -> bytes:
+        """Read the whole file at `path`; `read` adds the optional transfer cap."""
 
     @abstractmethod
     async def write(self, path: str, data: bytes) -> None:

{verifiers-0.2.2.dev74 → verifiers-0.2.2.dev75}/verifiers/v1/runtimes/docker/__init__.py

@@ -334,7 +334,7 @@ class DockerRuntime(Runtime):
         if run.exit_code != 0:
             raise SandboxError(f"docker exec -d failed: {run.stderr.strip()}")
 
-    async def read(self, path: str) -> bytes:
+    async def _read(self, path: str) -> bytes:
         proc = await asyncio.create_subprocess_exec(
             "docker",
             "exec",

{verifiers-0.2.2.dev74 → verifiers-0.2.2.dev75}/verifiers/v1/runtimes/modal.py

@@ -165,7 +165,7 @@ class ModalRuntime(Runtime):
             return path
         return f"{self.config.workdir.rstrip('/')}/{path}"
 
-    async def read(self, path: str) -> bytes:
+    async def _read(self, path: str) -> bytes:
         try:
             return await self._sandbox.filesystem.read_bytes.aio(self._abs(path))
         except Exception as e:

{verifiers-0.2.2.dev74 → verifiers-0.2.2.dev75}/verifiers/v1/runtimes/prime.py

@@ -270,7 +270,7 @@ class PrimeRuntime(Runtime):
                 f"prime background launch failed: {result.stderr.strip()}"
             )
 
-    async def read(self, path: str) -> bytes:
+    async def _read(self, path: str) -> bytes:
         # Avoid background-job log limits and base64 overhead by downloading binary data directly.
         # The temporary file is removed on every exit, and its byte read stays off the event loop.
         target = (

{verifiers-0.2.2.dev74 → verifiers-0.2.2.dev75}/verifiers/v1/runtimes/subprocess.py

@@ -96,7 +96,7 @@ class SubprocessRuntime(Runtime):
             proc
         )  # killed in stop() — a host process won't die on its own
 
-    async def read(self, path: str) -> bytes:
+    async def _read(self, path: str) -> bytes:
         return await asyncio.to_thread((self.workdir / path).read_bytes)
 
     async def write(self, path: str, data: bytes) -> None:

verifiers-0.2.2.dev75/verifiers/v1/tasksets/harbor/__init__.py

@@ -0,0 +1,16 @@
+from verifiers.v1.tasksets.harbor.env import HarborEnv, HarborEnvConfig
+from verifiers.v1.tasksets.harbor.taskset import (
+    HarborConfig,
+    HarborData,
+    HarborTask,
+    HarborTaskset,
+)
+
+__all__ = [
+    "HarborConfig",
+    "HarborData",
+    "HarborEnv",
+    "HarborEnvConfig",
+    "HarborTask",
+    "HarborTaskset",
+]

verifiers-0.2.2.dev75/verifiers/v1/tasksets/harbor/env.py

@@ -0,0 +1,124 @@
+"""The harbor taskset's own env: the single solver seat, plus separate-verifier
+grading for tasks that declare ``[verifier].environment_mode = "separate"``.
+
+The default env for harbor runs (the taskset package exports it). A shared-verifier
+task runs exactly as under the single-agent env: one `agent` trace, graded in the
+box it worked in. A separate-verifier task is graded by `finalize` instead: the
+solver's declared artifacts travel (collected by its task `finalize` while its box
+is alive), a fresh box is provisioned from the task's verifier declaration,
+`tests/` is staged there, and the verifier's rewards land on the solver's trace.
+No second agent is involved — the verifier is the task's own `tests/test.sh`.
+"""
+
+import asyncio
+import logging
+from contextlib import AsyncExitStack
+
+from pydantic import Field
+
+import verifiers.v1 as vf
+from verifiers.v1.runtimes import RuntimeConfig, provision_runtime
+from verifiers.v1.tasksets.harbor.taskset import (
+    HarborTask,
+    verifier_box_data,
+)
+from verifiers.v1.utils.artifacts import restore
+from verifiers.v1.utils.compile import resolve_runtime_config
+from verifiers.v1.utils.retries import backoff
+
+logger = logging.getLogger(__name__)
+
+
+class HarborEnvConfig(vf.EnvConfig):
+    agent: vf.AgentConfig = vf.AgentConfig()
+    """The one seat — the policy under evaluation/training; pin
+    `--env.agent.harness.*` to choose its program or runtime."""
+    verifier_runtime: RuntimeConfig | None = None
+    """Where a separate-verifier task grades. None derives the grading box from
+    the solver's runtime policy; set it (e.g. `--env.verifier-runtime.type prime
+    --env.verifier-runtime.vm true`) when the verifier needs different placement
+    than the agent."""
+    verifier_retries: int = Field(2, ge=0)
+    """Extra attempts at provisioning-and-grading the separate box before the
+    episode fails. Grading is deterministic; what these retry is the
+    infrastructure around it (image pulls, provisioning)."""
+
+
+class HarborEnv(vf.Env[HarborEnvConfig]):
+    async def run(self, task: vf.Task, agents: vf.Agents) -> None:
+        if not isinstance(task, HarborTask):
+            raise TypeError(
+                f"the harbor env runs harbor tasks; got {type(task).__name__}"
+            )
+        if task.data.verifier is None:
+            await agents.agent.run(task)
+            return
+        # Resolve the verifier's box before the solve, so an impossible pairing
+        # (e.g. a restricted Prime verifier without vm=true) costs nothing
+        # rather than a full agent run.
+        self._verifier_config(task)
+        await agents.agent.run(task.graded_elsewhere())
+
+    def _verifier_config(self, task: HarborTask) -> RuntimeConfig:
+        base = (
+            self.config.verifier_runtime
+            if self.config.verifier_runtime is not None
+            else self.config.agent.runtime
+        )
+        return resolve_runtime_config(base, HarborTask(verifier_box_data(task.data)))
+
+    async def finalize(self, task: vf.Task, episode: vf.Episode) -> None:
+        """Grade a separate-verifier task in its own box, onto the solver's trace.
+
+        Provision a fresh box from the task's verifier declaration, restore the
+        solver's collected artifacts, stage `tests/`, run the verifier, and record
+        its rewards (and any extra reward.json keys as metrics) on the solver's
+        trace. Infrastructure failures retry per `verifier_retries`; the last one
+        fails the episode — a grading box that can't be reached must never read
+        as reward 0."""
+        if not isinstance(task, HarborTask) or task.data.verifier is None:
+            return
+        solution = episode.traces[0]
+        if not solution.ok:
+            return
+        grader = HarborTask(verifier_box_data(task.data))
+        scores = await self._grade(self._verifier_config(task), grader, solution)
+        items = scores.items() if isinstance(scores, dict) else [("solved", scores)]
+        for name, value in items:
+            solution.record_reward(name, value)
+
+    async def _grade(
+        self, config: RuntimeConfig, grader: HarborTask, solution: vf.Trace
+    ) -> float | dict[str, float]:
+        last: Exception | None = None
+        for attempt in range(self.config.verifier_retries + 1):
+            if attempt:
+                delay = backoff(attempt - 1)
+                logger.warning(
+                    "harbor verifier attempt %d/%d failed (%s); retrying in %.1fs",
+                    attempt,
+                    self.config.verifier_retries + 1,
+                    last,
+                    delay,
+                )
+                await asyncio.sleep(delay)
+            try:
+                # The scoring deadline covers provisioning and grading, but not the
+                # box's teardown: a score already in hand must not be discarded
+                # because the teardown ran out the clock.
+                async with AsyncExitStack() as boxes:
+                    async with asyncio.timeout(grader.data.timeout.scoring):
+                        box = await boxes.enter_async_context(provision_runtime(config))
+                        await box.prepare_setup()
+                        # Artifacts first, tests second: an artifact entry pointing
+                        # into /tests must not survive staging, which wipes and
+                        # rebuilds that directory.
+                        await restore(box, solution.state.artifacts)
+                        await grader._stage_tests(box, wipe=True)
+                        await box.prepare_execution([])
+                        scores = await grader._graded(box, solution)
+                    return scores
+            except Exception as e:  # noqa: BLE001 - each attempt's failure is retried
+                last = e
+        assert last is not None
+        raise last