verifiers 0.3.2.dev44__tar.gz → 0.3.2.dev46__tar.gz

{verifiers-0.3.2.dev44 → verifiers-0.3.2.dev46}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.3.2.dev44
+Version: 0.3.2.dev46
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers
@@ -24,6 +24,8 @@ Requires-Python: <3.14,>=3.11
 Requires-Dist: aiohttp>=3.14.1
 Requires-Dist: anthropic>=1.0.0
 Requires-Dist: gepa>=0.0.6
+Requires-Dist: h11>=0.16.0
+Requires-Dist: httpx2>=2.12.0
 Requires-Dist: httpx>=0.27.0
 Requires-Dist: loguru>=0.7.0
 Requires-Dist: math-verify>=0.8.0
@@ -41,6 +43,7 @@ Requires-Dist: rich>=11.0.0
 Requires-Dist: tenacity>=8.5.0
 Requires-Dist: tomli-w>=1.0.0
 Requires-Dist: typing-extensions>=4.12.2
+Requires-Dist: uvicorn>=0.52.0
 Provides-Extra: harbor
 Requires-Dist: harbor==0.21.0; (python_full_version >= '3.12') and extra == 'harbor'
 Provides-Extra: lean

{verifiers-0.3.2.dev44 → verifiers-0.3.2.dev46}/pyproject.toml

@@ -40,6 +40,9 @@ dependencies = [
     "pyzmq>=27.1.0",
     "msgpack>=1.1.2",
     "httpx>=0.27.0",
+    "httpx2>=2.12.0",
+    "h11>=0.16.0",
+    "uvicorn>=0.52.0",
     "aiohttp>=3.14.1",
     "prime-pydantic-config[toml]>=0.4.3",
     "loguru>=0.7.0",

{verifiers-0.3.2.dev44 → verifiers-0.3.2.dev46}/tests/v1/test_scoring.py

@@ -89,6 +89,19 @@ async def test_config_plugged_fns_merge_and_override(tmp_path) -> None:
         HookTask(HookData(idx=0, prompt="abc"), config).hooks("reward")
 
 
+async def test_defer_scoring_defers_only_task_scoring() -> None:
+    task = HookTask(HookData(idx=0, prompt="abc"))
+    trace = vf.Trace(
+        agent=vf.AgentInfo(config=vf.AgentConfig()),
+        task=vf.TraceTask(type="HookTask", data=task.data),
+    )
+
+    await task.defer_scoring().score(trace)
+    assert trace.rewards == {}
+    await task.score(trace)
+    assert set(trace.rewards) == {"fmt", "lcs"}
+
+
 def test_compare_stdout_results_accepts_token_equal_text() -> None:
     assert vf.compare_stdout_results("hello   world\n", "hello world\n")
 

{verifiers-0.3.2.dev44 → verifiers-0.3.2.dev46}/verifiers/v1/agent.py

@@ -11,7 +11,7 @@ import asyncio
 import logging
 from collections.abc import AsyncIterator, Callable, Mapping
 from contextlib import asynccontextmanager, nullcontext
-from dataclasses import dataclass
+from dataclasses import dataclass, replace
 from typing import Self
 
 from verifiers.v1.clients import (
@@ -54,6 +54,27 @@ __all__ = ["Agent", "AgentConfig", "Agents", "TimeoutConfig", "make_agent"]
 logger = logging.getLogger(__name__)
 
 
+def resolve_rollout_timeouts(timeout: TimeoutConfig, task: Task) -> RolloutTimeouts:
+    """Apply an agent's stage-timeout precedence to one task."""
+    agent_timeout = (
+        timeout.rollout if timeout.rollout is not None else task.data.timeout.agent
+    )
+    return RolloutTimeouts(
+        setup=timeout.setup if timeout.setup is not None else task.data.timeout.setup,
+        agent=agent_timeout,
+        finalize=(
+            timeout.finalize
+            if timeout.finalize is not None
+            else task.data.timeout.finalize
+        ),
+        scoring=(
+            timeout.scoring
+            if timeout.scoring is not None
+            else task.data.timeout.scoring
+        ),
+    )
+
+
 def _check_borrowed_placement(
     task: Task, runtime: Runtime, base_config: RuntimeConfig
 ) -> None:
@@ -526,34 +547,15 @@ class Agent:
             runtime_config,
             tools=[*task.toolsets(task.config), *shared_tools.values()],
         )
-        # Timeout precedence: agent-level wins, else the task's, else no limit.
-        agent_timeout = (
-            self.timeout.rollout
-            if self.timeout.rollout is not None
-            else task.data.timeout.agent
-        )
+        timeouts = resolve_rollout_timeouts(self.timeout, task)
         return {
             "agent_config": self.config,
             "harness": self.harness,
             "ctx": self.ctx,
             "runtime_config": runtime_config,
-            "timeouts": RolloutTimeouts(
-                setup=(
-                    self.timeout.setup
-                    if self.timeout.setup is not None
-                    else task.data.timeout.setup
-                ),
-                agent=cap_remote_agent_timeout(agent_timeout, runtime_config, task),
-                finalize=(
-                    self.timeout.finalize
-                    if self.timeout.finalize is not None
-                    else task.data.timeout.finalize
-                ),
-                scoring=(
-                    self.timeout.scoring
-                    if self.timeout.scoring is not None
-                    else task.data.timeout.scoring
-                ),
+            "timeouts": replace(
+                timeouts,
+                agent=cap_remote_agent_timeout(timeouts.agent, runtime_config, task),
             ),
             "limits": self.limits,
             "shared_tools": shared_tools,

verifiers-0.3.2.dev46/verifiers/v1/envs/isolated_verifier/__init__.py

@@ -0,0 +1,7 @@
+from verifiers.v1.envs.isolated_verifier.env import (
+    IsolatedVerifierEnv,
+    IsolatedVerifierEnvConfig,
+    VerifierConfig,
+)
+
+__all__ = ["IsolatedVerifierEnv", "IsolatedVerifierEnvConfig", "VerifierConfig"]

verifiers-0.3.2.dev46/verifiers/v1/envs/isolated_verifier/env.py

@@ -0,0 +1,182 @@
+"""Deterministic task verification in a fresh runtime.
+
+One solver agent runs the task. Its task scoring is deferred, declared artifacts
+are collected after normal task finalization, and the solver runtime is destroyed.
+The task is then set up with a fresh controller in a fresh runtime, its artifacts
+are restored, and its ordinary metrics and rewards run there onto the solver's trace.
+"""
+
+import asyncio
+import copy
+import logging
+from contextlib import AsyncExitStack
+from pathlib import PurePosixPath
+from typing import Any
+
+from pydantic import Field, SerializationInfo, field_serializer
+
+import verifiers.v1 as vf
+from verifiers.v1.agent import resolve_rollout_timeouts
+from verifiers.v1.errors import TaskError, boundary
+from verifiers.v1.runtimes import Runtime, RuntimeConfig, provision_runtime
+from verifiers.v1.utils.compile import resolve_runtime_config
+from verifiers.v1.utils.decorators import invoke
+from verifiers.v1.utils.retries import backoff
+
+logger = logging.getLogger(__name__)
+
+
+class VerifierConfig(vf.BaseConfig):
+    runtime: RuntimeConfig | None = None
+    """Independent verifier placement and policy. None provisions a fresh runtime
+    equivalent to the solver's resolved task runtime."""
+    env: dict[str, str] | None = None
+    """Process environment for verifier setup and scoring. None uses the task's
+    normal runtime environment."""
+    retries: int = Field(2, ge=0)
+    """Extra fresh-runtime attempts after setup, restoration, staging, or scoring
+    failures."""
+
+    @field_serializer("runtime")
+    def serialize_runtime(
+        self, runtime: RuntimeConfig | None, info: SerializationInfo
+    ) -> dict | None:
+        """Keep an omitted image omitted across a resolved-config round trip."""
+        if runtime is None:
+            return None
+        values = runtime.model_dump(mode=info.mode)
+        if "image" not in runtime.model_fields_set:
+            values.pop("image", None)
+        return values
+
+
+class IsolatedVerifierEnvConfig(vf.EnvConfig):
+    agent: vf.AgentConfig = vf.AgentConfig()
+    """The one solver seat."""
+    verifier: VerifierConfig = VerifierConfig()
+    """The fresh verifier's runtime, process environment, and retry policy."""
+
+
+class IsolatedVerifierEnv(vf.Env[IsolatedVerifierEnvConfig]):
+    """Run one solver, then its deterministic task scoring in a fresh runtime."""
+
+    async def run(self, task: vf.Task, agents: vf.Agents) -> None:
+        if task.config.judges:
+            raise ValueError(
+                "isolated-verifier runs deterministic task metrics and rewards; "
+                "model-backed task judges are not supported"
+            )
+        self.verifier_config(task)  # Refuse an impossible verifier before solving.
+        await agents.agent.run(task.defer_scoring(), collect_artifacts=True)
+
+    def verifier_config(self, task: vf.Task) -> RuntimeConfig:
+        base = self.config.verifier.runtime or self.config.agent.runtime
+        config = resolve_runtime_config(base, task)
+        image_spec = type(base).model_fields.get("image")
+        if (
+            self.config.verifier.runtime is not None
+            and image_spec is not None
+            and "image" in base.model_fields_set
+        ):
+            config = config.model_copy(update={"image": base.image})
+        if isinstance(config, vf.SubprocessConfig):
+            raise TypeError(
+                "isolated-verifier requires a container runtime so artifacts can be "
+                "restored safely; configure the agent or verifier runtime as docker, "
+                "prime, or modal"
+            )
+        relative = [
+            artifact.source
+            for artifact in task.data.artifacts
+            if not PurePosixPath(artifact.source).is_absolute()
+        ]
+        solver = resolve_runtime_config(self.config.agent.runtime, task)
+        solver_workdir = PurePosixPath(getattr(solver, "workdir", "") or "/")
+        verifier_workdir = PurePosixPath(config.workdir)
+        if relative and solver_workdir != verifier_workdir:
+            raise ValueError(
+                "isolated-verifier cannot transfer relative artifacts "
+                f"{relative!r} between solver workdir {str(solver_workdir)!r} and "
+                f"verifier workdir {str(verifier_workdir)!r}; use matching workdirs "
+                "or absolute artifact paths"
+            )
+        return config
+
+    async def finalize(self, task: vf.Task, episode: vf.Episode) -> None:
+        solution = episode.traces[0]
+        if solution.ok:
+            graded = await self.grade(self.verifier_config(task), task, solution)
+            episode.traces[0] = graded[1]
+
+    async def stage_verifier(
+        self, task: vf.Task, solution: vf.Trace, runtime: Runtime
+    ) -> None:
+        artifacts = dict(solution.state.artifacts)
+        async with boundary(TaskError, "verifier task setup"):
+            await invoke(task.setup, {"trace": solution, "runtime": runtime})
+        await vf.restore(runtime, artifacts)
+        async with boundary(TaskError, "verifier staging"):
+            await invoke(task.stage_verifier, {"trace": solution, "runtime": runtime})
+
+    async def verify(self, task: vf.Task, solution: vf.Trace, runtime: Runtime) -> Any:
+        await task.score(solution, runtime)
+
+    async def grade(
+        self,
+        config: RuntimeConfig,
+        task: vf.Task,
+        solution: vf.Trace,
+        *,
+        scoring_timeout_covers_attempt: bool = False,
+    ) -> tuple[Any, vf.Trace]:
+        timeouts = resolve_rollout_timeouts(self.config.agent.timeout, task)
+        last: Exception | None = None
+        for attempt in range(self.config.verifier.retries + 1):
+            if attempt:
+                delay = backoff(attempt - 1)
+                logger.warning(
+                    "isolated verifier attempt %d/%d failed (%s); retrying in %.1fs",
+                    attempt,
+                    self.config.verifier.retries + 1,
+                    last,
+                    delay,
+                )
+                await asyncio.sleep(delay)
+            try:
+                # Teardown is outside the stage deadlines: a completed score must
+                # survive a slow cleanup of the verifier runtime.
+                async with (
+                    AsyncExitStack() as boxes,
+                    asyncio.timeout(
+                        timeouts.scoring if scoring_timeout_covers_attempt else None
+                    ),
+                ):
+                    async with asyncio.timeout(timeouts.setup):
+                        # Failed setup or scoring must not alter the next attempt.
+                        # Only the successful controller and trace leave this scope.
+                        verifier_task = copy.deepcopy(task)
+                        verifier_solution = copy.deepcopy(solution)
+                        runtime = await boxes.enter_async_context(
+                            provision_runtime(
+                                config,
+                                env=(
+                                    verifier_task.runtime_env()
+                                    if self.config.verifier.env is None
+                                    else self.config.verifier.env
+                                ),
+                            )
+                        )
+                        await runtime.prepare_setup()
+                        await self.stage_verifier(
+                            verifier_task, verifier_solution, runtime
+                        )
+                        await runtime.prepare_execution([])
+                    async with asyncio.timeout(timeouts.scoring):
+                        result = await self.verify(
+                            verifier_task, verifier_solution, runtime
+                        )
+                    return result, verifier_solution
+            except Exception as error:  # noqa: BLE001 - retry the whole fresh box
+                last = error
+        assert last is not None
+        raise last

{verifiers-0.3.2.dev44 → verifiers-0.3.2.dev46}/verifiers/v1/rollout.py

@@ -467,6 +467,7 @@ class Rollout:
                 if trace.timing.agent.start and not trace.timing.agent.end:
                     trace.timing.agent.end = time.time()
             if not self._failed and self._opened:
+                assert runtime is not None
                 trace.timing.finalize.start = time.time()
                 async with boundary(TaskError, "task finalize"):
                     async with asyncio.timeout(self._timeouts.finalize):

{verifiers-0.3.2.dev44 → verifiers-0.3.2.dev46}/verifiers/v1/task.py

@@ -173,15 +173,36 @@ class Task(Generic[DataT, StateT, ConfigT]):
     async def finalize(self, trace: Trace, runtime: Runtime) -> None:
         return None
 
+    async def stage_verifier(self, trace: Trace, runtime: Runtime) -> None:
+        """Prepare trusted verifier-only inputs after artifacts are restored."""
+        return
+
     async def validate(self, runtime: Runtime) -> bool | None:
         """Check the ground truth, or return None when no model-free check exists."""
         return None
 
+    def defer_scoring(self) -> Self:
+        """An independent copy whose task signals are deferred.
+
+        Lifecycle hooks still run normally: in particular, ``finalize`` can prepare
+        state before declared artifacts are collected and the solver runtime is
+        destroyed. Only task metrics, rewards, and judges are skipped; harness
+        metrics remain attached to the solver trace.
+        """
+        clone = copy.deepcopy(self)
+        clone.scoring_deferred = True
+        return clone
+
+    scoring_deferred: bool = False
+
     async def score(
         self,
         trace: Trace,
         runtime: Runtime | None = None,
     ) -> None:
+        if self.scoring_deferred:
+            return
+
         def requires_runtime(fn) -> bool:
             param = inspect.signature(fn).parameters.get("runtime")
             # A defaulted runtime parameter can still be called offline with None.

verifiers-0.3.2.dev46/verifiers/v1/tasksets/harbor/env.py

@@ -0,0 +1,82 @@
+"""The harbor taskset's own env: the single solver seat, plus separate-verifier
+grading for tasks that declare ``[verifier].environment_mode = "separate"``.
+
+The default env for harbor runs (the taskset package exports it). A shared-verifier
+task runs exactly as under the single-agent env: one `agent` trace, graded in the
+box it worked in. A separate-verifier task is graded by `finalize` instead: the
+solver's declared artifacts travel (collected after its task `finalize` while its
+box is alive), a fresh box is provisioned from the task's verifier declaration,
+`tests/` is staged there, and the verifier's rewards land on the solver's trace.
+No second agent is involved — the verifier is the task's own `tests/test.sh`.
+"""
+
+import verifiers.v1 as vf
+from verifiers.v1.envs.isolated_verifier import (
+    IsolatedVerifierEnv,
+    IsolatedVerifierEnvConfig,
+)
+from verifiers.v1.runtimes import Runtime, RuntimeConfig
+from verifiers.v1.tasksets.harbor.taskset import (
+    HarborTask,
+    verifier_box_data,
+)
+from verifiers.v1.utils.compile import resolve_runtime_config
+
+
+class HarborEnvConfig(IsolatedVerifierEnvConfig):
+    """The Harbor solver plus its optional independent verifier runtime."""
+
+
+class HarborEnv(IsolatedVerifierEnv, vf.Env[HarborEnvConfig]):
+    async def run(self, task: vf.Task, agents: vf.Agents) -> None:
+        if not isinstance(task, HarborTask):
+            raise TypeError(
+                f"the harbor env runs harbor tasks; got {type(task).__name__}"
+            )
+        if task.data.verifier is None:
+            await agents.agent.run(task)
+            return
+        # Resolve the verifier's box before the solve, so an impossible pairing
+        # (e.g. a restricted Prime verifier without vm=true) costs nothing
+        # rather than a full agent run.
+        self.verifier_config(task)
+        await agents.agent.run(task.defer_scoring(), collect_artifacts=True)
+
+    def verifier_config(self, task: HarborTask) -> RuntimeConfig:
+        base = (
+            self.config.verifier.runtime
+            if self.config.verifier.runtime is not None
+            else self.config.agent.runtime
+        )
+        return resolve_runtime_config(base, HarborTask(verifier_box_data(task.data)))
+
+    async def finalize(self, task: vf.Task, episode: vf.Episode) -> None:
+        """Grade a separate-verifier task in its own box, onto the solver's trace.
+
+        Provision a fresh box from the task's verifier declaration, restore the
+        solver's collected artifacts, stage `tests/`, run the verifier, and record
+        its rewards (and any extra reward.json keys as metrics) on the solver's
+        trace. Setup, restoration, staging, and scoring failures retry per
+        `verifier.retries`; the last one fails the episode."""
+        if not isinstance(task, HarborTask) or task.data.verifier is None:
+            return
+        solution = episode.traces[0]
+        if not solution.ok:
+            return
+        grader = HarborTask(verifier_box_data(task.data))
+        scores, solution = await self.grade(
+            self.verifier_config(task),
+            grader,
+            solution,
+            scoring_timeout_covers_attempt=True,
+        )
+        items = scores.items() if isinstance(scores, dict) else [("solved", scores)]
+        for name, value in items:
+            solution.record_reward(name, value)
+        episode.traces[0] = solution
+
+    async def verify(
+        self, task: vf.Task, solution: vf.Trace, runtime: Runtime
+    ) -> float | dict[str, float]:
+        assert isinstance(task, HarborTask)
+        return await task.run_verifier(runtime, solution)

{verifiers-0.3.2.dev44 → verifiers-0.3.2.dev46}/verifiers/v1/tasksets/harbor/taskset.py

@@ -16,7 +16,6 @@ otherwise build the verifier image from ``tests/Dockerfile``.
 """
 
 import asyncio
-import copy
 import hashlib
 import io
 import logging
@@ -163,6 +162,8 @@ class HarborData(TaskData):
 class HarborTask(Task[HarborData]):
     """Stage and run Harbor's verifier inside the task's live runtime."""
 
+    verifier_staged: bool = False
+
     def runtime_env(self) -> dict[str, str]:
         return resolve_env(self.data.env)
 
@@ -186,9 +187,9 @@ class HarborTask(Task[HarborData]):
                     f"{(result.stderr or result.stdout).strip()[-500:]}"
                 )
         if self.data.healthcheck is not None:
-            await self._wait_for_health(runtime, self.data.healthcheck)
+            await self.wait_for_health(runtime, self.data.healthcheck)
 
-    async def _wait_for_health(self, runtime: Runtime, healthcheck: dict) -> None:
+    async def wait_for_health(self, runtime: Runtime, healthcheck: dict) -> None:
         from harbor.environments.base import HealthcheckError
         from harbor.models.task.config import HealthcheckConfig
 
@@ -246,18 +247,15 @@ class HarborTask(Task[HarborData]):
                     f"collect hook failed (exit {result.exit_code}): "
                     f"{hook.command}\n{detail}"
                 )
-        trace.state.artifacts = await collect(runtime, self.data.artifacts)
-
-    def graded_elsewhere(self) -> "HarborTask":
-        """A copy whose `solved` records nothing here: the harbor env grades this
-        task's finished work in a separate box of the task's choosing."""
-        clone = copy.copy(self)
-        clone._graded_elsewhere = True
-        return clone
+        if not self.scoring_deferred:
+            trace.state.artifacts = await collect(runtime, self.data.artifacts)
 
-    _graded_elsewhere: bool = False
+    async def stage_verifier(self, runtime: Runtime) -> None:
+        # An artifact under /tests must not replace the task package's verifier.
+        await self.stage_tests(runtime, wipe=True)
+        self.verifier_staged = True
 
-    async def _stage_tests(self, runtime: Runtime, wipe: bool = False) -> None:
+    async def stage_tests(self, runtime: Runtime, wipe: bool = False) -> None:
         """Put the task package's `tests/` in `/tests`, where `test.sh` expects it.
 
         Raises rather than scoring stale state: a leftover reward file — planted by
@@ -286,24 +284,26 @@ class HarborTask(Task[HarborData]):
     @reward(weight=1.0)
     async def solved(self, runtime: Runtime, trace: Trace) -> float | dict[str, float]:
         if self.data.verifier is not None:
-            if not self._graded_elsewhere:
+            if not self.verifier_staged:
                 raise TaskError(
                     f"task {self.data.name!r} declares a separate verifier "
                     '([verifier].environment_mode = "separate"); grade it through '
                     "the harbor env (this taskset's default), or force shared "
                     "grading with --taskset.ignore-separate-verifier"
                 )
-            return {}
-        await self._stage_tests(runtime)
-        return await self._graded(runtime, trace)
+        else:
+            await self.stage_tests(runtime)
+        return await self.run_verifier(runtime, trace)
 
-    async def _graded(self, runtime: Runtime, trace: Trace) -> float | dict[str, float]:
+    async def run_verifier(
+        self, runtime: Runtime, trace: Trace
+    ) -> float | dict[str, float]:
         # By absolute path, in the runtime's configured workdir: Harbor execs the
         # script the same way, and scripts do grade the agent's work at `$PWD`.
         await runtime.run(
             ["bash", "/tests/test.sh"], resolve_env(self.data.verifier_env)
         )
-        scores = await self._reward_json(runtime)
+        scores = await self.read_reward_json(runtime)
         if scores is not None:
             if isinstance(scores, dict) and "reward" in scores:
                 trace.record_metrics(
@@ -325,7 +325,9 @@ class HarborTask(Task[HarborData]):
         except (SandboxError, OSError, ValueError):
             return 0.0
 
-    async def _reward_json(self, runtime: Runtime) -> float | dict[str, float] | None:
+    async def read_reward_json(
+        self, runtime: Runtime
+    ) -> float | dict[str, float] | None:
         """Read Harbor's scalar or keyed JSON reward, if it is valid.
 
         Bounded: this is a grading input, and nothing guarantees its size.