verifiers 0.2.2.dev73__tar.gz → 0.2.2.dev75__tar.gz

{verifiers-0.2.2.dev73 → verifiers-0.2.2.dev75}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.2.dev73
+Version: 0.2.2.dev75
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.2.2.dev73 → verifiers-0.2.2.dev75}/tests/v1/test_judges.py

@@ -461,7 +461,7 @@ async def test_error_attribution(monkeypatch, tmp_path):
         await JudgedTask(trace.task.data, taskset.config.task).score(
             trace, runtime=None
         )
-    assert "reference" not in trace.rewards
+    assert trace.rewards["reference"] is None  # seeded: expected but never scored
     assert len(trace.info["judge"]) == 1  # the billed call is still recorded
 
 

{verifiers-0.2.2.dev73 → verifiers-0.2.2.dev75}/tests/v1/test_trace.py

@@ -71,17 +71,22 @@ def test_wire_trace_round_trip():
         ],
     )
     tr.record_reward("r", 1.0)
+    tr.rewards.setdefault("solved", None)  # seeded: expected but never scored
+    tr.metrics.setdefault("acc", None)
     tr.info = {"build": "ok"}
     tr.stop("done")
 
     # the dump is plain pydantic — derived values are properties, so they're not serialized
     data = json.loads(tr.model_dump_json(exclude_none=True))
     assert "reward" not in data and "is_truncated" not in data
+    # exclude_none drops None FIELDS, not None dict values — unscored seeds survive
+    assert data["rewards"]["solved"] is None and data["metrics"]["acc"] is None
 
     rt = vf.WireTrace.model_validate(data)
     assert rt.num_branches == tr.num_branches == 2  # branch topology survived
     assert rt.num_turns == tr.num_turns == 2
-    assert rt.reward == 1.0  # property recomputed from `rewards`
+    assert rt.reward == 1.0  # property recomputed from `rewards`, seeds contribute 0
+    assert rt.rewards["solved"] is None
     assert rt.stop_condition == "done"
     assert rt.info == {"build": "ok"}
     assert (

{verifiers-0.2.2.dev73 → verifiers-0.2.2.dev75}/verifiers/v1/agent.py

@@ -29,7 +29,7 @@ from verifiers.v1.runtimes import (
     Runtime,
     RuntimeConfig,
     SubprocessConfig,
-    make_runtime,
+    provision_runtime,
     runtime_is_local,
 )
 from verifiers.v1.session import RolloutLimits
@@ -547,14 +547,8 @@ class Agent:
             if task is not None
             else self.runtime_config
         )
-        runtime = make_runtime(config)
-        try:
-            # start() inside the try: a failed start may already hold a remote
-            # sandbox, so it must reach stop() (safe on a partially-started runtime).
-            await runtime.start()
+        async with provision_runtime(config) as runtime:
             yield runtime
-        finally:
-            await runtime.stop()
 
 
 class _EpisodeAgent(Agent):

{verifiers-0.2.2.dev73 → verifiers-0.2.2.dev75}/verifiers/v1/cli/dashboard/eval.py

@@ -332,8 +332,8 @@ def Progress(
 
 def _score_segments(traces: list[Trace], source: str) -> str | None:
     """`name mean` segments for every reward/metric key seen across `traces`,
-    first-seen order (a trace records only the functions that ran for it, so keys
-    can vary); None when no trace recorded anything."""
+    first-seen order (a trace carries only its own task's signals — unscored ones
+    seeded as `None` — so keys can vary); None when no trace recorded anything."""
     names: list[str] = []
     for trace in traces:
         names.extend(n for n in getattr(trace, source) if n not in names)
@@ -347,11 +347,13 @@ def _score_segments(traces: list[Trace], source: str) -> str | None:
 
 
 def _score(trace: Trace, source: str, name: str) -> float:
-    """Rewards carry raw score + weight; the breakdown shows the raw score."""
+    """Rewards carry raw score + weight; the breakdown shows the raw score. A missing
+    or unscored (seeded `None`) entry reads as 0.0."""
     if source == "rewards":
         reward = trace.rewards.get(name)
         return reward.score if reward is not None else 0.0
-    return trace.metrics.get(name, 0.0)
+    value = trace.metrics.get(name)
+    return value if value is not None else 0.0
 
 
 def _breakdown(scored: list[Trace], done: list[Trace]) -> Table | None:

{verifiers-0.2.2.dev73 → verifiers-0.2.2.dev75}/verifiers/v1/envs/agentic_judge/env.py

@@ -398,7 +398,8 @@ class AgenticJudgeEnv(vf.Env[AgenticJudgeEnvConfig]):
             solution.record_metric(f"judge/{criterion.name}", scores[criterion.name])
         if self.config.score.task_weight != 1.0:
             for reward in solution.rewards.values():
-                reward.weight *= self.config.score.task_weight
+                if reward is not None:
+                    reward.weight *= self.config.score.task_weight
         total = sum(criterion.weight for criterion in criteria)
         reward = sum(c.weight * scores[c.name] for c in criteria) / total
         solution.record_reward("judge", reward, weight=self.config.score.judge_weight)

{verifiers-0.2.2.dev73 → verifiers-0.2.2.dev75}/verifiers/v1/harness.py

@@ -12,7 +12,12 @@ from verifiers.v1.errors import HarnessError, SandboxError, boundary
 from verifiers.v1.runtimes import ProgramResult, Runtime
 from verifiers.v1.task import TaskData
 from verifiers.v1.types import Messages
-from verifiers.v1.utils.decorators import discover_decorated, invoke_all
+from verifiers.v1.utils.decorators import (
+    discover_decorated,
+    invoke_all,
+    seed,
+    unseed,
+)
 
 if TYPE_CHECKING:
     # Annotation-only: `Trace` appears in signatures only, so this module stays
@@ -155,10 +160,12 @@ class Harness(ABC, Generic[ConfigT]):
         `runtime`) and can read what the harness left behind in the runtime."""
         available = {"task": trace.task.data, "trace": trace, "runtime": runtime}
         fns = discover_decorated(self, "metric")
+        seed(trace.metrics, (fn.__name__ for fn in fns))
         async with boundary(HarnessError, f"harness {self.config.id!r} metric"):
             results = await invoke_all(fns, available)
         for fn, result in zip(fns, results):
             if isinstance(result, Mapping):
+                unseed(trace.metrics, fn.__name__)
                 trace.record_metrics(result)
             else:
                 trace.record_metric(fn.__name__, result)

{verifiers-0.2.2.dev73 → verifiers-0.2.2.dev75}/verifiers/v1/runtimes/__init__.py

@@ -1,3 +1,5 @@
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager
 from typing import Annotated
 
 from pydantic import Field
@@ -45,6 +47,22 @@ def make_runtime(config: RuntimeConfig, name: str | None = None) -> Runtime:
     return runtime
 
 
+@asynccontextmanager
+async def provision_runtime(
+    config: RuntimeConfig, name: str | None = None
+) -> AsyncIterator[Runtime]:
+    """Provision a box from `config` and tear it down on exit.
+
+    `start()` sits inside the `try`: a failed start may already hold a paid sandbox, so
+    it has to reach `stop()` (which is safe on a partially-started runtime)."""
+    runtime = make_runtime(config, name)
+    try:
+        await runtime.start()
+        yield runtime
+    finally:
+        await runtime.stop()
+
+
 def runtime_is_local(config: RuntimeConfig) -> bool:
     """Whether a runtime of this config exchanges host-local URLs without a public
     tunnel, read off the runtime class without provisioning one."""
@@ -71,5 +89,6 @@ __all__ = [
     "SubprocessRuntime",
     "SubprocessRuntimeInfo",
     "make_runtime",
+    "provision_runtime",
     "runtime_is_local",
 ]

{verifiers-0.2.2.dev73 → verifiers-0.2.2.dev75}/verifiers/v1/runtimes/base.py

@@ -2,6 +2,7 @@
 
 import asyncio
 import atexit
+import base64
 import contextlib
 import hashlib
 import logging
@@ -285,9 +286,42 @@ class Runtime(ABC):
         argv = await self.prepare_uv_script(script, env)
         return await self.run([*argv, *(args or [])], env or {})
 
+    async def read(self, path: str, max_bytes: int | None = None) -> bytes:
+        """Read `path` into host memory. `max_bytes` caps the transfer, raising past
+        the cap — for a file written by something we don't control, whose size we
+        can't assume. The cap is enforced inside the box rather than after the
+        transfer, and base64 because `run` returns decoded text. Framework method —
+        override `_read`, not this."""
+        if max_bytes is None:
+            return await self._read(path)
+        # Through a temp file, not a pipe: `head | base64` exits with base64's 0
+        # even when the path is missing, and a missing file must raise here just
+        # as it does from `_read`.
+        result = await self.run(
+            [
+                "sh",
+                "-c",
+                (
+                    "t=$(mktemp) || exit 1; "
+                    'head -c "$1" -- "$2" > "$t" || { rm -f "$t"; exit 1; }; '
+                    'base64 < "$t"; rc=$?; rm -f "$t"; exit $rc'
+                ),
+                "sh",
+                str(max_bytes + 1),
+                path,
+            ],
+            {},
+        )
+        if result.exit_code:
+            raise SandboxError(f"read {path!r}: {result.stderr.strip()[-500:]}")
+        data = base64.b64decode(result.stdout)
+        if len(data) > max_bytes:
+            raise SandboxError(f"read {path!r}: over the {max_bytes} byte limit")
+        return data
+
     @abstractmethod
-    async def read(self, path: str) -> bytes:
-        pass
+    async def _read(self, path: str) -> bytes:
+        """Read the whole file at `path`; `read` adds the optional transfer cap."""
 
     @abstractmethod
     async def write(self, path: str, data: bytes) -> None:

{verifiers-0.2.2.dev73 → verifiers-0.2.2.dev75}/verifiers/v1/runtimes/docker/__init__.py

@@ -334,7 +334,7 @@ class DockerRuntime(Runtime):
         if run.exit_code != 0:
             raise SandboxError(f"docker exec -d failed: {run.stderr.strip()}")
 
-    async def read(self, path: str) -> bytes:
+    async def _read(self, path: str) -> bytes:
         proc = await asyncio.create_subprocess_exec(
             "docker",
             "exec",

{verifiers-0.2.2.dev73 → verifiers-0.2.2.dev75}/verifiers/v1/runtimes/modal.py

@@ -165,7 +165,7 @@ class ModalRuntime(Runtime):
             return path
         return f"{self.config.workdir.rstrip('/')}/{path}"
 
-    async def read(self, path: str) -> bytes:
+    async def _read(self, path: str) -> bytes:
         try:
             return await self._sandbox.filesystem.read_bytes.aio(self._abs(path))
         except Exception as e:

{verifiers-0.2.2.dev73 → verifiers-0.2.2.dev75}/verifiers/v1/runtimes/prime.py

@@ -270,7 +270,7 @@ class PrimeRuntime(Runtime):
                 f"prime background launch failed: {result.stderr.strip()}"
             )
 
-    async def read(self, path: str) -> bytes:
+    async def _read(self, path: str) -> bytes:
         # Avoid background-job log limits and base64 overhead by downloading binary data directly.
         # The temporary file is removed on every exit, and its byte read stays off the event loop.
         target = (

{verifiers-0.2.2.dev73 → verifiers-0.2.2.dev75}/verifiers/v1/runtimes/subprocess.py

@@ -96,7 +96,7 @@ class SubprocessRuntime(Runtime):
             proc
         )  # killed in stop() — a host process won't die on its own
 
-    async def read(self, path: str) -> bytes:
+    async def _read(self, path: str) -> bytes:
         return await asyncio.to_thread((self.workdir / path).read_bytes)
 
     async def write(self, path: str, data: bytes) -> None:

{verifiers-0.2.2.dev73 → verifiers-0.2.2.dev75}/verifiers/v1/task.py

@@ -23,7 +23,12 @@ from verifiers.v1.errors import TaskError, boundary
 from verifiers.v1.state import StateT
 from verifiers.v1.types import Messages, content_text
 from verifiers.v1.utils.artifacts import Artifact
-from verifiers.v1.utils.decorators import discover_decorated, invoke_all
+from verifiers.v1.utils.decorators import (
+    discover_decorated,
+    invoke_all,
+    seed,
+    unseed,
+)
 from verifiers.v1.utils.generic import concrete_type
 
 if TYPE_CHECKING:
@@ -180,31 +185,36 @@ class Task(Generic[DataT, StateT, ConfigT]):
                     judge for judge in judges if not requires_runtime(judge.score)
                 ]
 
+            seed(trace.metrics, (fn.__name__ for fn in metrics))
+            seed(trace.rewards, (fn.__name__ for fn in rewards))
+            seed(trace.rewards, (judge.reward_name for judge in judges))
+
             metric_results = await invoke_all(metrics, available)
             for fn, result in zip(metrics, metric_results):
                 if isinstance(result, Mapping):
+                    unseed(trace.metrics, fn.__name__)
                     trace.record_metrics(result)
                 else:
                     trace.record_metric(fn.__name__, result)
             reward_results = await invoke_all(rewards, available)
             for fn, result in zip(rewards, reward_results):
                 weight = getattr(fn, "_vf_weight", 1.0)
-                items = (
-                    result.items()
-                    if isinstance(result, Mapping)
-                    else [(fn.__name__, result)]
-                )
+                if isinstance(result, Mapping):
+                    unseed(trace.rewards, fn.__name__)
+                    items = result.items()
+                else:
+                    items = [(fn.__name__, result)]
                 for key, value in items:
                     trace.record_reward(key, value, weight)
             judge_results = await invoke_all(
                 [judge.score for judge in judges], available
             )
             for judge, result in zip(judges, judge_results):
-                items = (
-                    result.items()
-                    if isinstance(result, Mapping)
-                    else [(judge.reward_name, result)]
-                )
+                if isinstance(result, Mapping):
+                    unseed(trace.rewards, judge.reward_name)
+                    items = result.items()
+                else:
+                    items = [(judge.reward_name, result)]
                 for key, value in items:
                     trace.record_reward(key, value, judge.config.weight)
 

verifiers-0.2.2.dev75/verifiers/v1/tasksets/harbor/__init__.py

@@ -0,0 +1,16 @@
+from verifiers.v1.tasksets.harbor.env import HarborEnv, HarborEnvConfig
+from verifiers.v1.tasksets.harbor.taskset import (
+    HarborConfig,
+    HarborData,
+    HarborTask,
+    HarborTaskset,
+)
+
+__all__ = [
+    "HarborConfig",
+    "HarborData",
+    "HarborEnv",
+    "HarborEnvConfig",
+    "HarborTask",
+    "HarborTaskset",
+]

verifiers-0.2.2.dev75/verifiers/v1/tasksets/harbor/env.py

@@ -0,0 +1,124 @@
+"""The harbor taskset's own env: the single solver seat, plus separate-verifier
+grading for tasks that declare ``[verifier].environment_mode = "separate"``.
+
+The default env for harbor runs (the taskset package exports it). A shared-verifier
+task runs exactly as under the single-agent env: one `agent` trace, graded in the
+box it worked in. A separate-verifier task is graded by `finalize` instead: the
+solver's declared artifacts travel (collected by its task `finalize` while its box
+is alive), a fresh box is provisioned from the task's verifier declaration,
+`tests/` is staged there, and the verifier's rewards land on the solver's trace.
+No second agent is involved — the verifier is the task's own `tests/test.sh`.
+"""
+
+import asyncio
+import logging
+from contextlib import AsyncExitStack
+
+from pydantic import Field
+
+import verifiers.v1 as vf
+from verifiers.v1.runtimes import RuntimeConfig, provision_runtime
+from verifiers.v1.tasksets.harbor.taskset import (
+    HarborTask,
+    verifier_box_data,
+)
+from verifiers.v1.utils.artifacts import restore
+from verifiers.v1.utils.compile import resolve_runtime_config
+from verifiers.v1.utils.retries import backoff
+
+logger = logging.getLogger(__name__)
+
+
+class HarborEnvConfig(vf.EnvConfig):
+    agent: vf.AgentConfig = vf.AgentConfig()
+    """The one seat — the policy under evaluation/training; pin
+    `--env.agent.harness.*` to choose its program or runtime."""
+    verifier_runtime: RuntimeConfig | None = None
+    """Where a separate-verifier task grades. None derives the grading box from
+    the solver's runtime policy; set it (e.g. `--env.verifier-runtime.type prime
+    --env.verifier-runtime.vm true`) when the verifier needs different placement
+    than the agent."""
+    verifier_retries: int = Field(2, ge=0)
+    """Extra attempts at provisioning-and-grading the separate box before the
+    episode fails. Grading is deterministic; what these retry is the
+    infrastructure around it (image pulls, provisioning)."""
+
+
+class HarborEnv(vf.Env[HarborEnvConfig]):
+    async def run(self, task: vf.Task, agents: vf.Agents) -> None:
+        if not isinstance(task, HarborTask):
+            raise TypeError(
+                f"the harbor env runs harbor tasks; got {type(task).__name__}"
+            )
+        if task.data.verifier is None:
+            await agents.agent.run(task)
+            return
+        # Resolve the verifier's box before the solve, so an impossible pairing
+        # (e.g. a restricted Prime verifier without vm=true) costs nothing
+        # rather than a full agent run.
+        self._verifier_config(task)
+        await agents.agent.run(task.graded_elsewhere())
+
+    def _verifier_config(self, task: HarborTask) -> RuntimeConfig:
+        base = (
+            self.config.verifier_runtime
+            if self.config.verifier_runtime is not None
+            else self.config.agent.runtime
+        )
+        return resolve_runtime_config(base, HarborTask(verifier_box_data(task.data)))
+
+    async def finalize(self, task: vf.Task, episode: vf.Episode) -> None:
+        """Grade a separate-verifier task in its own box, onto the solver's trace.
+
+        Provision a fresh box from the task's verifier declaration, restore the
+        solver's collected artifacts, stage `tests/`, run the verifier, and record
+        its rewards (and any extra reward.json keys as metrics) on the solver's
+        trace. Infrastructure failures retry per `verifier_retries`; the last one
+        fails the episode — a grading box that can't be reached must never read
+        as reward 0."""
+        if not isinstance(task, HarborTask) or task.data.verifier is None:
+            return
+        solution = episode.traces[0]
+        if not solution.ok:
+            return
+        grader = HarborTask(verifier_box_data(task.data))
+        scores = await self._grade(self._verifier_config(task), grader, solution)
+        items = scores.items() if isinstance(scores, dict) else [("solved", scores)]
+        for name, value in items:
+            solution.record_reward(name, value)
+
+    async def _grade(
+        self, config: RuntimeConfig, grader: HarborTask, solution: vf.Trace
+    ) -> float | dict[str, float]:
+        last: Exception | None = None
+        for attempt in range(self.config.verifier_retries + 1):
+            if attempt:
+                delay = backoff(attempt - 1)
+                logger.warning(
+                    "harbor verifier attempt %d/%d failed (%s); retrying in %.1fs",
+                    attempt,
+                    self.config.verifier_retries + 1,
+                    last,
+                    delay,
+                )
+                await asyncio.sleep(delay)
+            try:
+                # The scoring deadline covers provisioning and grading, but not the
+                # box's teardown: a score already in hand must not be discarded
+                # because the teardown ran out the clock.
+                async with AsyncExitStack() as boxes:
+                    async with asyncio.timeout(grader.data.timeout.scoring):
+                        box = await boxes.enter_async_context(provision_runtime(config))
+                        await box.prepare_setup()
+                        # Artifacts first, tests second: an artifact entry pointing
+                        # into /tests must not survive staging, which wipes and
+                        # rebuilds that directory.
+                        await restore(box, solution.state.artifacts)
+                        await grader._stage_tests(box, wipe=True)
+                        await box.prepare_execution([])
+                        scores = await grader._graded(box, solution)
+                    return scores
+            except Exception as e:  # noqa: BLE001 - each attempt's failure is retried
+                last = e
+        assert last is not None
+        raise last