verifiers 0.2.2.dev73__py3-none-any.whl → 0.2.2.dev75__py3-none-any.whl

verifiers/v1/agent.py

@@ -29,7 +29,7 @@ from verifiers.v1.runtimes import (
     Runtime,
     RuntimeConfig,
     SubprocessConfig,
-    make_runtime,
+    provision_runtime,
     runtime_is_local,
 )
 from verifiers.v1.session import RolloutLimits
@@ -547,14 +547,8 @@ class Agent:
             if task is not None
             else self.runtime_config
         )
-        runtime = make_runtime(config)
-        try:
-            # start() inside the try: a failed start may already hold a remote
-            # sandbox, so it must reach stop() (safe on a partially-started runtime).
-            await runtime.start()
+        async with provision_runtime(config) as runtime:
             yield runtime
-        finally:
-            await runtime.stop()
 
 
 class _EpisodeAgent(Agent):

verifiers/v1/cli/dashboard/eval.py

@@ -332,8 +332,8 @@ def Progress(
 
 def _score_segments(traces: list[Trace], source: str) -> str | None:
     """`name mean` segments for every reward/metric key seen across `traces`,
-    first-seen order (a trace records only the functions that ran for it, so keys
-    can vary); None when no trace recorded anything."""
+    first-seen order (a trace carries only its own task's signals — unscored ones
+    seeded as `None` — so keys can vary); None when no trace recorded anything."""
     names: list[str] = []
     for trace in traces:
         names.extend(n for n in getattr(trace, source) if n not in names)
@@ -347,11 +347,13 @@ def _score_segments(traces: list[Trace], source: str) -> str | None:
 
 
 def _score(trace: Trace, source: str, name: str) -> float:
-    """Rewards carry raw score + weight; the breakdown shows the raw score."""
+    """Rewards carry raw score + weight; the breakdown shows the raw score. A missing
+    or unscored (seeded `None`) entry reads as 0.0."""
     if source == "rewards":
         reward = trace.rewards.get(name)
         return reward.score if reward is not None else 0.0
-    return trace.metrics.get(name, 0.0)
+    value = trace.metrics.get(name)
+    return value if value is not None else 0.0
 
 
 def _breakdown(scored: list[Trace], done: list[Trace]) -> Table | None:

verifiers/v1/envs/agentic_judge/env.py

@@ -398,7 +398,8 @@ class AgenticJudgeEnv(vf.Env[AgenticJudgeEnvConfig]):
             solution.record_metric(f"judge/{criterion.name}", scores[criterion.name])
         if self.config.score.task_weight != 1.0:
             for reward in solution.rewards.values():
-                reward.weight *= self.config.score.task_weight
+                if reward is not None:
+                    reward.weight *= self.config.score.task_weight
         total = sum(criterion.weight for criterion in criteria)
         reward = sum(c.weight * scores[c.name] for c in criteria) / total
         solution.record_reward("judge", reward, weight=self.config.score.judge_weight)

verifiers/v1/harness.py

@@ -12,7 +12,12 @@ from verifiers.v1.errors import HarnessError, SandboxError, boundary
 from verifiers.v1.runtimes import ProgramResult, Runtime
 from verifiers.v1.task import TaskData
 from verifiers.v1.types import Messages
-from verifiers.v1.utils.decorators import discover_decorated, invoke_all
+from verifiers.v1.utils.decorators import (
+    discover_decorated,
+    invoke_all,
+    seed,
+    unseed,
+)
 
 if TYPE_CHECKING:
     # Annotation-only: `Trace` appears in signatures only, so this module stays
@@ -155,10 +160,12 @@ class Harness(ABC, Generic[ConfigT]):
         `runtime`) and can read what the harness left behind in the runtime."""
         available = {"task": trace.task.data, "trace": trace, "runtime": runtime}
         fns = discover_decorated(self, "metric")
+        seed(trace.metrics, (fn.__name__ for fn in fns))
         async with boundary(HarnessError, f"harness {self.config.id!r} metric"):
             results = await invoke_all(fns, available)
         for fn, result in zip(fns, results):
             if isinstance(result, Mapping):
+                unseed(trace.metrics, fn.__name__)
                 trace.record_metrics(result)
             else:
                 trace.record_metric(fn.__name__, result)

verifiers/v1/runtimes/__init__.py

@@ -1,3 +1,5 @@
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager
 from typing import Annotated
 
 from pydantic import Field
@@ -45,6 +47,22 @@ def make_runtime(config: RuntimeConfig, name: str | None = None) -> Runtime:
     return runtime
 
 
+@asynccontextmanager
+async def provision_runtime(
+    config: RuntimeConfig, name: str | None = None
+) -> AsyncIterator[Runtime]:
+    """Provision a box from `config` and tear it down on exit.
+
+    `start()` sits inside the `try`: a failed start may already hold a paid sandbox, so
+    it has to reach `stop()` (which is safe on a partially-started runtime)."""
+    runtime = make_runtime(config, name)
+    try:
+        await runtime.start()
+        yield runtime
+    finally:
+        await runtime.stop()
+
+
 def runtime_is_local(config: RuntimeConfig) -> bool:
     """Whether a runtime of this config exchanges host-local URLs without a public
     tunnel, read off the runtime class without provisioning one."""
@@ -71,5 +89,6 @@ __all__ = [
     "SubprocessRuntime",
     "SubprocessRuntimeInfo",
     "make_runtime",
+    "provision_runtime",
     "runtime_is_local",
 ]

verifiers/v1/runtimes/base.py

@@ -2,6 +2,7 @@
 
 import asyncio
 import atexit
+import base64
 import contextlib
 import hashlib
 import logging
@@ -285,9 +286,42 @@ class Runtime(ABC):
         argv = await self.prepare_uv_script(script, env)
         return await self.run([*argv, *(args or [])], env or {})
 
+    async def read(self, path: str, max_bytes: int | None = None) -> bytes:
+        """Read `path` into host memory. `max_bytes` caps the transfer, raising past
+        the cap — for a file written by something we don't control, whose size we
+        can't assume. The cap is enforced inside the box rather than after the
+        transfer, and base64 because `run` returns decoded text. Framework method —
+        override `_read`, not this."""
+        if max_bytes is None:
+            return await self._read(path)
+        # Through a temp file, not a pipe: `head | base64` exits with base64's 0
+        # even when the path is missing, and a missing file must raise here just
+        # as it does from `_read`.
+        result = await self.run(
+            [
+                "sh",
+                "-c",
+                (
+                    "t=$(mktemp) || exit 1; "
+                    'head -c "$1" -- "$2" > "$t" || { rm -f "$t"; exit 1; }; '
+                    'base64 < "$t"; rc=$?; rm -f "$t"; exit $rc'
+                ),
+                "sh",
+                str(max_bytes + 1),
+                path,
+            ],
+            {},
+        )
+        if result.exit_code:
+            raise SandboxError(f"read {path!r}: {result.stderr.strip()[-500:]}")
+        data = base64.b64decode(result.stdout)
+        if len(data) > max_bytes:
+            raise SandboxError(f"read {path!r}: over the {max_bytes} byte limit")
+        return data
+
     @abstractmethod
-    async def read(self, path: str) -> bytes:
-        pass
+    async def _read(self, path: str) -> bytes:
+        """Read the whole file at `path`; `read` adds the optional transfer cap."""
 
     @abstractmethod
     async def write(self, path: str, data: bytes) -> None:

verifiers/v1/runtimes/docker/__init__.py

@@ -334,7 +334,7 @@ class DockerRuntime(Runtime):
         if run.exit_code != 0:
             raise SandboxError(f"docker exec -d failed: {run.stderr.strip()}")
 
-    async def read(self, path: str) -> bytes:
+    async def _read(self, path: str) -> bytes:
         proc = await asyncio.create_subprocess_exec(
             "docker",
             "exec",

verifiers/v1/runtimes/modal.py

@@ -165,7 +165,7 @@ class ModalRuntime(Runtime):
             return path
         return f"{self.config.workdir.rstrip('/')}/{path}"
 
-    async def read(self, path: str) -> bytes:
+    async def _read(self, path: str) -> bytes:
         try:
             return await self._sandbox.filesystem.read_bytes.aio(self._abs(path))
         except Exception as e:

verifiers/v1/runtimes/prime.py

@@ -270,7 +270,7 @@ class PrimeRuntime(Runtime):
                 f"prime background launch failed: {result.stderr.strip()}"
             )
 
-    async def read(self, path: str) -> bytes:
+    async def _read(self, path: str) -> bytes:
         # Avoid background-job log limits and base64 overhead by downloading binary data directly.
         # The temporary file is removed on every exit, and its byte read stays off the event loop.
         target = (

verifiers/v1/runtimes/subprocess.py

@@ -96,7 +96,7 @@ class SubprocessRuntime(Runtime):
             proc
         )  # killed in stop() — a host process won't die on its own
 
-    async def read(self, path: str) -> bytes:
+    async def _read(self, path: str) -> bytes:
         return await asyncio.to_thread((self.workdir / path).read_bytes)
 
     async def write(self, path: str, data: bytes) -> None:

verifiers/v1/task.py

@@ -23,7 +23,12 @@ from verifiers.v1.errors import TaskError, boundary
 from verifiers.v1.state import StateT
 from verifiers.v1.types import Messages, content_text
 from verifiers.v1.utils.artifacts import Artifact
-from verifiers.v1.utils.decorators import discover_decorated, invoke_all
+from verifiers.v1.utils.decorators import (
+    discover_decorated,
+    invoke_all,
+    seed,
+    unseed,
+)
 from verifiers.v1.utils.generic import concrete_type
 
 if TYPE_CHECKING:
@@ -180,31 +185,36 @@ class Task(Generic[DataT, StateT, ConfigT]):
                     judge for judge in judges if not requires_runtime(judge.score)
                 ]
 
+            seed(trace.metrics, (fn.__name__ for fn in metrics))
+            seed(trace.rewards, (fn.__name__ for fn in rewards))
+            seed(trace.rewards, (judge.reward_name for judge in judges))
+
             metric_results = await invoke_all(metrics, available)
             for fn, result in zip(metrics, metric_results):
                 if isinstance(result, Mapping):
+                    unseed(trace.metrics, fn.__name__)
                     trace.record_metrics(result)
                 else:
                     trace.record_metric(fn.__name__, result)
             reward_results = await invoke_all(rewards, available)
             for fn, result in zip(rewards, reward_results):
                 weight = getattr(fn, "_vf_weight", 1.0)
-                items = (
-                    result.items()
-                    if isinstance(result, Mapping)
-                    else [(fn.__name__, result)]
-                )
+                if isinstance(result, Mapping):
+                    unseed(trace.rewards, fn.__name__)
+                    items = result.items()
+                else:
+                    items = [(fn.__name__, result)]
                 for key, value in items:
                     trace.record_reward(key, value, weight)
             judge_results = await invoke_all(
                 [judge.score for judge in judges], available
             )
             for judge, result in zip(judges, judge_results):
-                items = (
-                    result.items()
-                    if isinstance(result, Mapping)
-                    else [(judge.reward_name, result)]
-                )
+                if isinstance(result, Mapping):
+                    unseed(trace.rewards, judge.reward_name)
+                    items = result.items()
+                else:
+                    items = [(judge.reward_name, result)]
                 for key, value in items:
                     trace.record_reward(key, value, judge.config.weight)
 

verifiers/v1/tasksets/harbor/__init__.py

@@ -1,3 +1,4 @@
+from verifiers.v1.tasksets.harbor.env import HarborEnv, HarborEnvConfig
 from verifiers.v1.tasksets.harbor.taskset import (
     HarborConfig,
     HarborData,
@@ -5,4 +6,11 @@ from verifiers.v1.tasksets.harbor.taskset import (
     HarborTaskset,
 )
 
-__all__ = ["HarborConfig", "HarborData", "HarborTask", "HarborTaskset"]
+__all__ = [
+    "HarborConfig",
+    "HarborData",
+    "HarborEnv",
+    "HarborEnvConfig",
+    "HarborTask",
+    "HarborTaskset",
+]

verifiers/v1/tasksets/harbor/env.py

@@ -0,0 +1,124 @@
+"""The harbor taskset's own env: the single solver seat, plus separate-verifier
+grading for tasks that declare ``[verifier].environment_mode = "separate"``.
+
+The default env for harbor runs (the taskset package exports it). A shared-verifier
+task runs exactly as under the single-agent env: one `agent` trace, graded in the
+box it worked in. A separate-verifier task is graded by `finalize` instead: the
+solver's declared artifacts travel (collected by its task `finalize` while its box
+is alive), a fresh box is provisioned from the task's verifier declaration,
+`tests/` is staged there, and the verifier's rewards land on the solver's trace.
+No second agent is involved — the verifier is the task's own `tests/test.sh`.
+"""
+
+import asyncio
+import logging
+from contextlib import AsyncExitStack
+
+from pydantic import Field
+
+import verifiers.v1 as vf
+from verifiers.v1.runtimes import RuntimeConfig, provision_runtime
+from verifiers.v1.tasksets.harbor.taskset import (
+    HarborTask,
+    verifier_box_data,
+)
+from verifiers.v1.utils.artifacts import restore
+from verifiers.v1.utils.compile import resolve_runtime_config
+from verifiers.v1.utils.retries import backoff
+
+logger = logging.getLogger(__name__)
+
+
+class HarborEnvConfig(vf.EnvConfig):
+    agent: vf.AgentConfig = vf.AgentConfig()
+    """The one seat — the policy under evaluation/training; pin
+    `--env.agent.harness.*` to choose its program or runtime."""
+    verifier_runtime: RuntimeConfig | None = None
+    """Where a separate-verifier task grades. None derives the grading box from
+    the solver's runtime policy; set it (e.g. `--env.verifier-runtime.type prime
+    --env.verifier-runtime.vm true`) when the verifier needs different placement
+    than the agent."""
+    verifier_retries: int = Field(2, ge=0)
+    """Extra attempts at provisioning-and-grading the separate box before the
+    episode fails. Grading is deterministic; what these retry is the
+    infrastructure around it (image pulls, provisioning)."""
+
+
+class HarborEnv(vf.Env[HarborEnvConfig]):
+    async def run(self, task: vf.Task, agents: vf.Agents) -> None:
+        if not isinstance(task, HarborTask):
+            raise TypeError(
+                f"the harbor env runs harbor tasks; got {type(task).__name__}"
+            )
+        if task.data.verifier is None:
+            await agents.agent.run(task)
+            return
+        # Resolve the verifier's box before the solve, so an impossible pairing
+        # (e.g. a restricted Prime verifier without vm=true) costs nothing
+        # rather than a full agent run.
+        self._verifier_config(task)
+        await agents.agent.run(task.graded_elsewhere())
+
+    def _verifier_config(self, task: HarborTask) -> RuntimeConfig:
+        base = (
+            self.config.verifier_runtime
+            if self.config.verifier_runtime is not None
+            else self.config.agent.runtime
+        )
+        return resolve_runtime_config(base, HarborTask(verifier_box_data(task.data)))
+
+    async def finalize(self, task: vf.Task, episode: vf.Episode) -> None:
+        """Grade a separate-verifier task in its own box, onto the solver's trace.
+
+        Provision a fresh box from the task's verifier declaration, restore the
+        solver's collected artifacts, stage `tests/`, run the verifier, and record
+        its rewards (and any extra reward.json keys as metrics) on the solver's
+        trace. Infrastructure failures retry per `verifier_retries`; the last one
+        fails the episode — a grading box that can't be reached must never read
+        as reward 0."""
+        if not isinstance(task, HarborTask) or task.data.verifier is None:
+            return
+        solution = episode.traces[0]
+        if not solution.ok:
+            return
+        grader = HarborTask(verifier_box_data(task.data))
+        scores = await self._grade(self._verifier_config(task), grader, solution)
+        items = scores.items() if isinstance(scores, dict) else [("solved", scores)]
+        for name, value in items:
+            solution.record_reward(name, value)
+
+    async def _grade(
+        self, config: RuntimeConfig, grader: HarborTask, solution: vf.Trace
+    ) -> float | dict[str, float]:
+        last: Exception | None = None
+        for attempt in range(self.config.verifier_retries + 1):
+            if attempt:
+                delay = backoff(attempt - 1)
+                logger.warning(
+                    "harbor verifier attempt %d/%d failed (%s); retrying in %.1fs",
+                    attempt,
+                    self.config.verifier_retries + 1,
+                    last,
+                    delay,
+                )
+                await asyncio.sleep(delay)
+            try:
+                # The scoring deadline covers provisioning and grading, but not the
+                # box's teardown: a score already in hand must not be discarded
+                # because the teardown ran out the clock.
+                async with AsyncExitStack() as boxes:
+                    async with asyncio.timeout(grader.data.timeout.scoring):
+                        box = await boxes.enter_async_context(provision_runtime(config))
+                        await box.prepare_setup()
+                        # Artifacts first, tests second: an artifact entry pointing
+                        # into /tests must not survive staging, which wipes and
+                        # rebuilds that directory.
+                        await restore(box, solution.state.artifacts)
+                        await grader._stage_tests(box, wipe=True)
+                        await box.prepare_execution([])
+                        scores = await grader._graded(box, solution)
+                    return scores
+            except Exception as e:  # noqa: BLE001 - each attempt's failure is retried
+                last = e
+        assert last is not None
+        raise last

verifiers/v1/tasksets/harbor/taskset.py

@@ -1,18 +1,25 @@
 """Harbor tasksets backed by Harbor Hub packages.
 
 The Harbor CLI downloads and caches each task directory. Its verifier runs in the
-same runtime the harness edited, then writes the score to
+runtime the harness edited — or, when the task asks for it with
+``[verifier].environment_mode = "separate"``, in a second box the agent never
+touched, carrying only what the task declared — the harbor env provisions and
+grades that box (see ``env.py``). Either way the score lands in
 ``/logs/verifier/reward.json`` or the legacy ``reward.txt``.
 
 A pullable ``[environment].docker_image`` becomes ``TaskData.image``. Verifiers does
 not build Dockerfile-only environments, so those are rejected unless ``ignore_dockerfile``
 deliberately uses the harness runtime image. Tasks without an environment also use that
-image unless ``require_image`` is set.
+image unless ``require_image`` is set. The same rule applies to a declared
+``[verifier.environment]``: it needs a pullable ``docker_image``, since Harbor would
+otherwise build the verifier image from ``tests/Dockerfile``.
 """
 
 import asyncio
+import copy
 import hashlib
 import io
+import logging
 import shutil
 import subprocess
 import sys
@@ -26,7 +33,7 @@ from typing import Annotated
 from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError
 
 from verifiers.v1.configs.taskset import TasksetConfig
-from verifiers.v1.errors import SandboxError
+from verifiers.v1.errors import SandboxError, TaskError
 from verifiers.v1.runtimes import Runtime
 from verifiers.v1.task import Task, TaskData, TaskResources, TaskTimeout
 from verifiers.v1.taskset import Taskset
@@ -34,9 +41,12 @@ from verifiers.v1.trace import Trace
 from verifiers.v1.utils.artifacts import Artifact, collect
 from verifiers.v1.utils.decorators import reward
 
+logger = logging.getLogger(__name__)
+
 CACHE = Path.home() / ".cache" / "harbor"
 HARBOR_INSTALL_HINT = "uv sync --python 3.12 --extra harbor"
 REWARD_JSON = "/logs/verifier/reward.json"
+MAX_REWARD_BYTES = 1024 * 1024
 REWARD_JSON_ADAPTER = TypeAdapter(
     float | Annotated[dict[str, float], Field(min_length=1)],
     config=ConfigDict(strict=True, allow_inf_nan=False),
@@ -76,6 +86,11 @@ class HarborConfig(TasksetConfig):
     instead of rejecting it. The Dockerfile is NOT built, so the task scores against the
     harness image rather than its declared environment — only correct when that image already
     has what the task needs (e.g. you've pointed the runtime at the right image)."""
+    ignore_separate_verifier: bool = False
+    """Grade every task in the agent's own box, even one whose `[verifier]` asks for a
+    separate one. Trades the isolation for a sandbox per task; useful when provisioning
+    is the bottleneck. Note what it gives up: the grader becomes reachable by the agent
+    that just ran."""
 
 
 class Author(BaseModel):
@@ -90,6 +105,27 @@ class CollectHook(BaseModel):
     timeout_sec: float = 600.0
 
 
+class VerifierConfig(BaseModel):
+    """The box this task's verifier wants, when it wants one of its own.
+
+    `None` on `HarborData` means shared — grade where the agent worked, which is still
+    Harbor's default and every task that says nothing."""
+
+    image: str | None = None
+    """Pullable ref from `[verifier.environment].docker_image`. None keeps the task's
+    own image, which is what Harbor's fresh copy of `[environment]` resolves to."""
+    resources: TaskResources = TaskResources()
+    workdir: str | None = None
+    fresh_copy: bool = False
+    """Whether this came from Harbor's fresh copy of `[environment]` rather than a
+    declared `[verifier.environment]`. A fresh copy inherits the agent box's resolved
+    resources; a declared environment states its own, and what it omits falls back to
+    the run's rather than to the agent's task-derived values."""
+    network_allow: list[str] = Field(default_factory=lambda: ["*"])
+    """Destinations the verifier may reach, from the verifier's network mode. `["*"]`
+    is unrestricted; `[]` is Harbor's `no-network` / `allow_internet = false`."""
+
+
 class HarborData(TaskData):
     """Parsed ``task.toml`` metadata plus the host-side verifier directory.
 
@@ -111,6 +147,9 @@ class HarborData(TaskData):
     collect: list[CollectHook] = Field(default_factory=list)
     """`[[verifier.collect]]` blocks: commands that snapshot runtime state into files
     after the agent stops, so the files can travel to a grading box as artifacts."""
+    verifier: VerifierConfig | None = None
+    """The verifier's own box, when `[verifier].environment_mode` asks for one. None
+    grades in the agent's box."""
 
 
 class HarborTask(Task[HarborData]):
@@ -145,30 +184,59 @@ class HarborTask(Task[HarborData]):
                 )
         trace.state.artifacts = await collect(runtime, self.data.artifacts)
 
-    @reward(weight=1.0)
-    async def solved(self, runtime: Runtime, trace: Trace) -> float | dict[str, float]:
+    def graded_elsewhere(self) -> "HarborTask":
+        """A copy whose `solved` records nothing here: the harbor env grades this
+        task's finished work in a separate box of the task's choosing."""
+        clone = copy.copy(self)
+        clone._graded_elsewhere = True
+        return clone
+
+    _graded_elsewhere: bool = False
+
+    async def _stage_tests(self, runtime: Runtime, wipe: bool = False) -> None:
+        """Put the task package's `tests/` in `/tests`, where `test.sh` expects it.
+
+        Raises rather than scoring stale state: a leftover reward file — planted by
+        the agent or shipped in the image — must be gone before `test.sh` runs, so a
+        removal that fails must not fall through to reading it.
+
+        `wipe` for a box we did not watch being built: a fresh container of the task's
+        image can ship its own `/tests`, and a leftover file there would be graded as
+        though it came from the package.
+        """
         await runtime.write(
             "/tmp/tests.tgz", make_tar(Path(self.data.task_dir) / "tests")
         )
-        await runtime.run(
-            [
-                "sh",
-                "-c",
-                "mkdir -p /logs/verifier /tests && tar -xzf /tmp/tests.tgz -C /tests",
-            ],
-            {},
-        )
-        await runtime.run(
-            [
-                "sh",
-                "-c",
-                (
-                    "rm -f /logs/verifier/reward.json /logs/verifier/reward.txt"
-                    " && cd /tests && bash test.sh"
-                ),
-            ],
-            verifier_env(self.data),
+        stage = (
+            f"{'rm -rf /tests && ' if wipe else ''}"
+            "rm -f /logs/verifier/reward.json /logs/verifier/reward.txt && "
+            "mkdir -p /logs/verifier /tests && tar -xzf /tmp/tests.tgz -C /tests"
         )
+        result = await runtime.run(["sh", "-c", stage], {})
+        if result.exit_code:
+            raise TaskError(
+                f"staging tests failed (exit {result.exit_code}): "
+                f"{(result.stderr or result.stdout).strip()[-500:]}"
+            )
+
+    @reward(weight=1.0)
+    async def solved(self, runtime: Runtime, trace: Trace) -> float | dict[str, float]:
+        if self.data.verifier is not None:
+            if not self._graded_elsewhere:
+                raise TaskError(
+                    f"task {self.data.name!r} declares a separate verifier "
+                    '([verifier].environment_mode = "separate"); grade it through '
+                    "the harbor env (this taskset's default), or force shared "
+                    "grading with --taskset.ignore-separate-verifier"
+                )
+            return {}
+        await self._stage_tests(runtime)
+        return await self._graded(runtime, trace)
+
+    async def _graded(self, runtime: Runtime, trace: Trace) -> float | dict[str, float]:
+        # By absolute path, in the runtime's configured workdir: Harbor execs the
+        # script the same way, and scripts do grade the agent's work at `$PWD`.
+        await runtime.run(["bash", "/tests/test.sh"], verifier_env(self.data))
         scores = await self._reward_json(runtime)
         if scores is not None:
             if isinstance(scores, dict) and "reward" in scores:
@@ -178,19 +246,75 @@ class HarborTask(Task[HarborData]):
                 return {"reward": scores["reward"]}
             return scores
         try:
-            reward = (await runtime.read("/logs/verifier/reward.txt")).decode().strip()
+            reward = (
+                (
+                    await runtime.read(
+                        "/logs/verifier/reward.txt", max_bytes=MAX_REWARD_BYTES
+                    )
+                )
+                .decode()
+                .strip()
+            )
             return float(reward or 0)
         except (SandboxError, OSError, ValueError):
             return 0.0
 
     async def _reward_json(self, runtime: Runtime) -> float | dict[str, float] | None:
-        """Read Harbor's scalar or keyed JSON reward, if it is valid."""
+        """Read Harbor's scalar or keyed JSON reward, if it is valid.
+
+        Bounded: this is a grading input, and nothing guarantees its size.
+        """
         try:
-            return REWARD_JSON_ADAPTER.validate_json(await runtime.read(REWARD_JSON))
+            return REWARD_JSON_ADAPTER.validate_json(
+                await runtime.read(REWARD_JSON, max_bytes=MAX_REWARD_BYTES)
+            )
         except (SandboxError, OSError, ValidationError):
             return None
 
 
+def verifier_box_data(data: HarborData) -> HarborData:
+    """The verifier's box, declared as task data — the harbor env resolves the
+    grading runtime from it (image, workdir, resources, network policy), exactly
+    as the solver's box resolves from the solver task's.
+
+    Which box follows Harbor: a declared `[verifier.environment]` states its own
+    image, workdir, and resources, and what it omits is the run's default; a
+    fresh copy of `[environment]` keeps the task's own. The verifier's network
+    policy applies either way."""
+    verifier = data.verifier
+    if verifier is None:
+        raise TaskError(f"task {data.name!r} declares no separate verifier")
+    fresh = verifier.fresh_copy
+    return data.model_copy(
+        update={
+            "name": f"{data.name} (verifier)",
+            "image": verifier.image if verifier.image is not None else data.image,
+            "workdir": data.workdir if fresh else verifier.workdir,
+            "resources": data.resources if fresh else verifier.resources,
+            "network_allow": list(verifier.network_allow),
+            "network_block": [],
+        }
+    )
+
+
+def task_resources(environment, multiplier: float) -> TaskResources:
+    """Harbor environment resource requests, scaled, as `TaskResources`.
+
+    Harbor declares CPU counts and MB sizes; `TaskResources` wants counts and GB.
+    GPU requests are never scaled.
+    """
+    return TaskResources(
+        cpu=environment.cpus * multiplier if environment.cpus else None,
+        memory=environment.memory_mb / 1024 * multiplier
+        if environment.memory_mb
+        else None,
+        gpu=str(environment.gpus) if environment.gpus else None,
+        disk=environment.storage_mb / 1024 * multiplier
+        if environment.storage_mb
+        else None,
+    )
+
+
 def harbor_cli() -> str:
     scripts_dir = Path(sys.executable).parent
     harbor_bin = shutil.which("harbor", path=str(scripts_dir))
@@ -318,7 +442,7 @@ def parse_task(task_dir: Path, idx: int, harbor_config: HarborConfig) -> HarborD
 
     harbor_task = HarborModelTask(task_dir)
     parsed = harbor_task.config
-    artifacts, collect = parse_verifier_extras(task_dir, parsed)
+    artifacts, hooks, verifier = parse_verifier_extras(task_dir, parsed, harbor_config)
     environment = parsed.environment
     network = parsed.agent.explicit_phase_policy() or environment.resolve_baseline()
     task, meta = parsed.task, parsed.metadata
@@ -368,18 +492,7 @@ def parse_task(task_dir: Path, idx: int, harbor_config: HarborConfig) -> HarborD
             if scoring_timeout is not None
             else None,
         ),
-        resources=TaskResources(
-            cpu=environment.cpus * harbor_config.resource_multiplier
-            if environment.cpus
-            else None,
-            memory=environment.memory_mb / 1024 * harbor_config.resource_multiplier
-            if environment.memory_mb
-            else None,
-            gpu=str(environment.gpus) if environment.gpus else None,
-            disk=environment.storage_mb / 1024 * harbor_config.resource_multiplier
-            if environment.storage_mb
-            else None,
-        ),
+        resources=task_resources(environment, harbor_config.resource_multiplier),
         keywords=task.keywords if task else [],
         authors=authors,
         difficulty=meta.get("difficulty"),
@@ -388,14 +501,22 @@ def parse_task(task_dir: Path, idx: int, harbor_config: HarborConfig) -> HarborD
         task_dir=str(task_dir),
         verifier_env=parsed.verifier.env,
         artifacts=artifacts,
-        collect=collect,
+        collect=hooks,
+        verifier=verifier,
     )
 
 
 def parse_verifier_extras(
-    task_dir: Path, parsed
-) -> tuple[list[Artifact], list[CollectHook]]:
-    """Parse supported artifact and collect-hook settings."""
+    task_dir: Path, parsed, harbor_config: HarborConfig
+) -> tuple[list[Artifact], list[CollectHook], VerifierConfig | None]:
+    """Harbor's `artifacts`, `[[verifier.collect]]` blocks, and verifier environment,
+    narrowed to what verifiers' verifier-runtime integration can honor.
+
+    The convention dir is deliberately not prepended here (Harbor's
+    `with_convention_entry` would): collection injects it itself, as an optional sweep.
+    Prepending it would make it an explicitly declared entry, and declared entries are
+    required — which would fail every task that never writes there.
+    """
     from harbor.constants import MAIN_SERVICE_NAME
     from harbor.models.task.artifacts import (
         effective_artifact_service,
@@ -403,13 +524,6 @@ def parse_verifier_extras(
     )
 
     verifier = parsed.verifier
-    if verifier.environment is not None:
-        raise ValueError(
-            f"{task_dir.name}: [verifier.environment] declares a separate verifier "
-            "image. Grading runs in a fresh box built from the task's own image, so "
-            "only the agent's delta has to travel; a different verifier image needs "
-            "the full working tree copied over and isn't supported yet."
-        )
     if verifier.user is not None:
         raise ValueError(f"{task_dir.name}: [verifier].user is not supported")
 
@@ -447,7 +561,89 @@ def parse_verifier_extras(
             )
         hooks.append(CollectHook(command=hook.command, timeout_sec=hook.timeout_sec))
 
-    return artifacts, hooks
+    return artifacts, hooks, parse_verifier_environment(task_dir, parsed, harbor_config)
+
+
+def parse_verifier_environment(
+    task_dir: Path, parsed, harbor_config: HarborConfig
+) -> VerifierConfig | None:
+    """The box Harbor wants this task's verifier in, or None to grade in the agent's.
+
+    Harbor resolves `[verifier.environment]` if declared, else a deep copy of
+    `[environment]` — so a mode-only `separate` lands on the task's own image and needs
+    nothing but a second box. A declared environment is the case that can name a
+    different image, and the case that can name none at all: there Harbor builds
+    `tests/Dockerfile`, which verifiers never does.
+    """
+    from harbor.models.task.config import NetworkMode, TaskOS
+    from harbor.models.task.verifier_mode import (
+        VerifierEnvironmentMode,
+        resolve_effective_verifier_env_config,
+        resolve_task_verifier_mode,
+    )
+
+    if resolve_task_verifier_mode(parsed) != VerifierEnvironmentMode.SEPARATE:
+        return None
+    if harbor_config.ignore_separate_verifier:
+        logger.warning(
+            "%s: asks for a separate verifier; grading in the agent's box anyway "
+            "(--taskset.ignore-separate-verifier)",
+            task_dir.name,
+        )
+        return None
+
+    environment = resolve_effective_verifier_env_config(parsed, None)
+    if environment is None:  # unreachable while the mode is SEPARATE
+        raise ValueError(f"{task_dir.name}: separate verifier resolved no environment")
+    declared = parsed.verifier.environment is not None
+
+    if declared and environment.docker_image is None:
+        if not harbor_config.ignore_dockerfile:
+            raise ValueError(
+                f"{task_dir.name}: [verifier.environment] names no docker_image, so "
+                "Harbor would build the verifier image from tests/Dockerfile. Verifiers "
+                "pulls images and never builds them: build and push it yourself (e.g. "
+                "`prime images push`) and set [verifier.environment].docker_image to the "
+                "resulting ref, or pass --taskset.ignore-dockerfile to grade in the "
+                "agent's image instead."
+            )
+        logger.warning(
+            "%s: [verifier.environment] names no docker_image — grading in the agent's "
+            "image rather than building tests/Dockerfile, so the verifier runs somewhere "
+            "the task never declared",
+            task_dir.name,
+        )
+    unsupported = [
+        field
+        for field in ("healthcheck", "mcp_servers", "skills_dir", "gpu_types", "tpu")
+        if getattr(environment, field, None)
+    ]
+    if environment.os != TaskOS.LINUX or unsupported:
+        raise ValueError(
+            f"{task_dir.name}: verifier environment declares "
+            f"{unsupported or environment.os}, which verifiers' verifier-runtime "
+            "integration cannot honor"
+        )
+
+    network = parsed.verifier.explicit_phase_policy() or environment.resolve_baseline()
+    return VerifierConfig(
+        image=environment.docker_image if declared else None,
+        # A declared environment states its own resources; what it leaves out is the
+        # run's default, not the agent task's. A fresh copy is the task's environment,
+        # so it keeps whatever the agent box resolved to.
+        resources=(
+            task_resources(environment, harbor_config.resource_multiplier)
+            if declared
+            else TaskResources()
+        ),
+        workdir=environment.workdir if declared else None,
+        fresh_copy=not declared,
+        network_allow=(
+            ["*"]
+            if network.network_mode == NetworkMode.PUBLIC
+            else list(network.allowed_hosts)
+        ),
+    )
 
 
 def verifier_env(task: HarborData) -> dict[str, str]:

verifiers/v1/trace.py

@@ -319,10 +319,11 @@ class Trace(BaseModel, Generic[DataT, StateT, AgentConfigT]):
     calls: list[ModelCall] = Field(default_factory=list)
     """Every model call; automatically recorded at intercept time + linked into `nodes`."""
 
-    rewards: dict[str, Reward] = Field(default_factory=dict)
-    """Named, weighted rewards"""
-    metrics: dict[str, float] = Field(default_factory=dict)
-    """Unweighted, named metrics"""
+    rewards: dict[str, Reward | None] = Field(default_factory=dict)
+    """Named, weighted rewards; `None` means scoring didn't run (e.g. because of a
+    preceding error)."""
+    metrics: dict[str, float | None] = Field(default_factory=dict)
+    """Unweighted, named metrics; `None` as in `rewards`."""
     info: dict[str, Any] = Field(default_factory=dict)
     """Scratch space for task-specific metadata."""
     state: StateT = Field(default_factory=State, exclude=True)
@@ -346,7 +347,7 @@ class Trace(BaseModel, Generic[DataT, StateT, AgentConfigT]):
 
     @property
     def reward(self) -> float:
-        return sum(r.value for r in self.rewards.values())
+        return sum(r.value for r in self.rewards.values() if r is not None)
 
     @property
     def has_error(self) -> bool:

verifiers/v1/utils/decorators.py

@@ -2,7 +2,7 @@
 
 import asyncio
 import inspect
-from collections.abc import Callable
+from collections.abc import Callable, Iterable
 from typing import Any, TypeVar, overload
 
 F = TypeVar("F", bound=Callable[..., Any])
@@ -38,6 +38,22 @@ async def invoke_all(
     return await asyncio.gather(*(invoke(fn, available) for fn in fns))
 
 
+def seed(scores: dict[str, Any], names: Iterable[str]) -> None:
+    """Mark every expected scoring key as unscored (`None`) before invocation, so a
+    trace records which signals should have run even when scoring fails midway.
+    Overwrites: a re-scoring attempt resets its own names, so stale values from a
+    previous attempt never read as fresh."""
+    for name in names:
+        scores[name] = None
+
+
+def unseed(scores: dict[str, Any], name: str) -> None:
+    """Drop a still-unscored seed — a handler returning keyed scores records under
+    its result's keys, not its function name."""
+    if scores.get(name) is None:
+        scores.pop(name, None)
+
+
 def mark(attr: str, **extra: Any) -> Callable[[F], F]:
     def decorator(f: F) -> F:
         setattr(f, attr, True)

verifiers/v1/utils/platform.py

@@ -93,7 +93,8 @@ def trace_to_sample(
     # Flatten sub-rewards to top-level keys the way v0 does (raw scores, as v0's
     # per-function outputs were); env metrics stay nested.
     for name, reward in trace.rewards.items():
-        sample.setdefault(name, reward.score)
+        if reward is not None:
+            sample.setdefault(name, reward.score)
     return sample
 
 
@@ -128,8 +129,15 @@ def _run_metrics(episodes: list[Episode], traces: list[Trace]) -> dict[str, Any]
     sums: dict[str, float] = {}
     counts: dict[str, int] = {}
     for trace in scored:
-        scores = {name: reward.score for name, reward in trace.rewards.items()}
-        for name, value in {**scores, **trace.metrics}.items():
+        scores = {
+            name: reward.score
+            for name, reward in trace.rewards.items()
+            if reward is not None
+        }
+        metrics = {
+            name: value for name, value in trace.metrics.items() if value is not None
+        }
+        for name, value in {**scores, **metrics}.items():
             sums[name] = sums.get(name, 0.0) + value
             counts[name] = counts.get(name, 0) + 1
     n = len(scored)

{verifiers-0.2.2.dev73.dist-info → verifiers-0.2.2.dev75.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.2.dev73
+Version: 0.2.2.dev75
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.2.2.dev73.dist-info → verifiers-0.2.2.dev75.dist-info}/RECORD

@@ -168,20 +168,20 @@ verifiers/utils/tool_utils.py,sha256=gInWZODWQUZN4TyEMuuITyOUm9qlHJxtcvr1zZl_zJs
 verifiers/utils/usage_utils.py,sha256=Mw6WC78r6EsLyRFdP9ICveqptkjizbSkAC86ZH0geVU,3918
 verifiers/utils/version_utils.py,sha256=39gV8-mNF9GspAgKloLo8oy7_MVluMrwGPrntQv2hx4,2656
 verifiers/v1/__init__.py,sha256=kacTRsGsRHfOruv16ZyIOesfvCH_nW98ARKPYSgcNIc,7509
-verifiers/v1/agent.py,sha256=3kQNASeO0aedCN8m95RAzxu2TbK0rfG7Ix6-f2yxQ7Q,31545
+verifiers/v1/agent.py,sha256=MQVKGjUhrP8uzRXAq94uJHkEsdHFziPxQHZ7TOD324I,31306
 verifiers/v1/env.py,sha256=g6fpoG-Z9lM3vTKrrU-QRJY3S88_y9zsPHwBvfs5vdY,17865
 verifiers/v1/episode.py,sha256=YtuRTV_PpwDglXFeN6UNg8IswpP1HseFLCYtascGMNI,3218
 verifiers/v1/errors.py,sha256=kQeEPX06TwAIuiz7MlWFYirrpKbx5STi6A5b_dtQQoA,6885
 verifiers/v1/graph.py,sha256=aBAzh1Ibp3klKNYjvzNyJSbjV8Oi5TJ791dN7o6ex1E,29070
-verifiers/v1/harness.py,sha256=i3pYHmdVPD7RtfZ-MAHlctaoLO8FxazCPmNy7mtPoSk,10954
+verifiers/v1/harness.py,sha256=eKuqvYbQjnCMavkiYWZQDHJgH08m6SJ4PuevPuShjS0,11097
 verifiers/v1/judge.py,sha256=Pvr0C41ah1qkNSZ0WXdkDvBMfv5YSfY1REHncwlJ8cg,9346
 verifiers/v1/legacy.py,sha256=eLsFiwihOMh4zS_u1V7DXF22r9beoG4k2JGwNEcSPfg,22872
 verifiers/v1/rollout.py,sha256=s1f1VZAAAI89j3y9ceq_E4mvX0F7cFQAnyx27SZsjvA,17881
 verifiers/v1/session.py,sha256=tNAraQfiJKpWqgBFNLyPhfHiw0JhzzkyOfRcKWGnR2Q,6939
 verifiers/v1/state.py,sha256=EckF2bWp-vV4b1jYJ9sLI5xrfGuI5spIgYYwW926toI,595
-verifiers/v1/task.py,sha256=D5PKxu6WlqCjv7hE311aaNvBA3QWVWDuPIEwDsq-J7g,9075
+verifiers/v1/task.py,sha256=jwMKiKlMtksTd8j2dcDlFtb6LC8BRE-N5jkNXlMp2jc,9457
 verifiers/v1/taskset.py,sha256=fp2E0IEhL_Ybj9cegZwljfTmW27p_30zTWHFKw_kXE4,4374
-verifiers/v1/trace.py,sha256=_bGHOrE6fRLGm0JYc0PrrrO3lyr-qqOF5Y1bHfViwR0,19347
+verifiers/v1/trace.py,sha256=Z4uK-lYAbl-6CHFhTo-gadHnbi63LC0sJtUERC_m5ss,19477
 verifiers/v1/types.py,sha256=1PxamJspmoTc3OlFZAwH6o_2i_Tg9ZNvHxttIHWIxq8,8948
 verifiers/v1/acp/__init__.py,sha256=9SYCFtzGUM_wH98ldpVLcFhoq2M999jbG0tU5ODY70U,2297
 verifiers/v1/acp/_runner.py,sha256=BcYaNZHuawzCgxOVdhiF6PY_B1HMxIA1MwHy0mOzhSk,7740
@@ -196,7 +196,7 @@ verifiers/v1/cli/serve.py,sha256=_mdJaHVVe4pXzcpAot0ugcmiN3ra2gS4GrGmLghnX5M,286
 verifiers/v1/cli/validate.py,sha256=7ax6FqIzNBSYjd3JXK4v7Vbq34Rozyh1OvGpYnGqlSg,10266
 verifiers/v1/cli/dashboard/__init__.py,sha256=v-baMxQuWxOCsbU7-p_jj2Q9BUnTN-TWi1q2hK6rU2s,198
 verifiers/v1/cli/dashboard/base.py,sha256=kUP93zJSIVLptSbnWX6MOy8kGg63fpeAzPqakCpPyDc,3547
-verifiers/v1/cli/dashboard/eval.py,sha256=nQ24ptQ_6ylmrB9VoZkgga1CLp_7sXZueC_MC4O9Dow,33452
+verifiers/v1/cli/dashboard/eval.py,sha256=4j2YCPyzNHDT7d8MwqYR3C906lzwDVAd3FxgFYFgsw0,33588
 verifiers/v1/cli/dashboard/replay.py,sha256=CvRVaf0dUum5v0IzPQbFPPikBQmTrkMeaunGfVMTWHc,2755
 verifiers/v1/cli/dashboard/validate.py,sha256=rLsQ_31DjTIZpC_VO7zSG41ncUEBxqCx8mL9nTnGiMc,3650
 verifiers/v1/cli/eval/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -234,7 +234,7 @@ verifiers/v1/dialects/chat.py,sha256=R_otqi5N0snB1QVKIYloNeLHR9kc7OmI06HYg1ir_Lw
 verifiers/v1/dialects/responses.py,sha256=ismnxUikPcPUIm80FUTPBbj-Cn1e0yGAsISMEseAWRc,13679
 verifiers/v1/envs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 verifiers/v1/envs/agentic_judge/__init__.py,sha256=6vwtMCQ_cuvuubCF5-nrE5W7gGgzE-LIPKorkM5DXdw,309
-verifiers/v1/envs/agentic_judge/env.py,sha256=jreQPnbp8pJQSyZW0Me2IW-JipYsczqgohfM4-JCz9c,16968
+verifiers/v1/envs/agentic_judge/env.py,sha256=6QrfuIlg-9MkcqIn5_6F3qJ6ddLJNaeYsZRJC9EzGkw,17011
 verifiers/v1/envs/best_of_n/__init__.py,sha256=mubASiwXMNIhnQFDccNAc7yZ0OFsOVK8hH8WIx9b2A4,119
 verifiers/v1/envs/best_of_n/env.py,sha256=KEUJBCVqblnXP_lEwm3mjGdaikBMkecz8UjOFPBhxc0,1987
 verifiers/v1/envs/single_agent/__init__.py,sha256=r0edAc_6gtHBhPzeBGVcdHZZqSdCPZnLMje6eVgyz5U,138
@@ -297,13 +297,13 @@ verifiers/v1/mcp/__init__.py,sha256=pm8ZWBiL1q1tY4wHvJzM_QrZ9oR8aGm-bIRWiRJsgHI,
 verifiers/v1/mcp/launch.py,sha256=wBpyeiOO86l1H2lVMbLVSV9Lbp90XeyXqauuK8snfEU,20171
 verifiers/v1/mcp/server.py,sha256=dRBxytFgS6rv1BS6t5EgBd3xeF6U1nTyt0ArDCLk3VE,12301
 verifiers/v1/mcp/toolset.py,sha256=T8Ioiq4wphxVOTHUDf28ZoQBM2ZYEevGnGgw8TFEMiA,1002
-verifiers/v1/runtimes/__init__.py,sha256=TqNtJVahzQuLS4fgCVeT9xdOzTkp0hRAPOkl7eFVDUg,2011
-verifiers/v1/runtimes/base.py,sha256=sTCiLo51yAU_T-Voc3BTH-FuzyEM0cGY05VzFE4DmYc,14720
+verifiers/v1/runtimes/__init__.py,sha256=Kwe612RjtGzrQhc19QdXc7CmZAnQLH-EdK_xqkKP3Tk,2629
+verifiers/v1/runtimes/base.py,sha256=tZRELSfZmi3l_1BsjabnNC3mEPfqNvUNevdnRvLPv6w,16257
 verifiers/v1/runtimes/limiters.py,sha256=ZjjYUFEt20yH8ggL5uU9w9U500AOB44qSUTQ7bYyh0c,2730
-verifiers/v1/runtimes/modal.py,sha256=FnocoJd9lQVQS3V3CoBon7KpKXm1bKWWeU2FAkq2bKU,8825
-verifiers/v1/runtimes/prime.py,sha256=zMMoB85uqwXjRTTyb3q5C7Xs4VmHagB8oI92Kuc-E1E,14755
-verifiers/v1/runtimes/subprocess.py,sha256=cs985D8-n_DOk88IRTyP1vfVlyUV1NBvS450lC-vw_s,5987
-verifiers/v1/runtimes/docker/__init__.py,sha256=eC5hjfyPwvwEKaeZUeP9EN8tJkrl0HUxGMzjiK4tK4Q,14555
+verifiers/v1/runtimes/modal.py,sha256=eFiT-Ovf_RwQZbSw3Dwv2EXeHV0d6uGvPswqKYKCw5k,8826
+verifiers/v1/runtimes/prime.py,sha256=kwvPqUI8D-GvcFwYRdpTHE1hXS0vfJGPlFOLgzm4Mvs,14756
+verifiers/v1/runtimes/subprocess.py,sha256=ZrHRlTuTyCrcSKpiPD2ElcPpe97EvVsXJ8BfwUaNXiI,5988
+verifiers/v1/runtimes/docker/__init__.py,sha256=vyEgf6XgAwjGgBicJchJvNgcDgYXrRvyQC4W52-23GU,14556
 verifiers/v1/runtimes/docker/egress.py,sha256=37UXgNX9gnoLxjMg4KZaJIGoApCAM6MK22eJGNoMi38,14514
 verifiers/v1/serve/__init__.py,sha256=sr_zkDzAHnOcQm-QRm8xwJBmCMFD-xhKIFPaDZvMbUs,641
 verifiers/v1/serve/client.py,sha256=CxpOwCn4ehuCXfqDJeV9zj7O0XC7MPcI-OZ1L_CSDlo,7057
@@ -311,8 +311,9 @@ verifiers/v1/serve/pool.py,sha256=AyOMjDYi9PLGA8Z-bnr6Fd5cnubuRavEBuudbVWtq2E,14
 verifiers/v1/serve/server.py,sha256=eFWJsTUDbMywKQXKDuk57QvS7Y4IZwW2Ok3LwJHMFLI,9200
 verifiers/v1/serve/types.py,sha256=3F4IIGIQgYITlEFxeX3uNJYrJlKugFrPTozbDNwDfBY,2674
 verifiers/v1/tasksets/__init__.py,sha256=b62O4WDNbGfJH_4NjYmznn5otWovdfvUgJZyoIZyLh8,596
-verifiers/v1/tasksets/harbor/__init__.py,sha256=wwVGA8N7E2Q2qA5WGHuLRrn4DgG5oaRHrCS2RW3NXVY,195
-verifiers/v1/tasksets/harbor/taskset.py,sha256=9rfWTg0s3zPAEppZnjh7zNyi2EbM-qORK2lyWQZQL7s,19565
+verifiers/v1/tasksets/harbor/__init__.py,sha256=JxThzHFYEkuSfQbVf3pEnjordgdfe4MU-QOsSSbyovU,326
+verifiers/v1/tasksets/harbor/env.py,sha256=bNF5dfbhcsdrd_8JmW5ved8Yk-mHhrOgKkVEA563qEo,5847
+verifiers/v1/tasksets/harbor/taskset.py,sha256=vmDjP7dhxchZ5-J9IUeiATwwchxUvAeciCevD6W_N4c,29032
 verifiers/v1/tasksets/lean/__init__.py,sha256=oyv-GbJQBvCyGwo7Ysxm2NjZW-aX4j1uDoHU0f-E7xQ,767
 verifiers/v1/tasksets/lean/scoring.py,sha256=sfzsT6MUK0zdMgAQ57_P_0CBaPHy3M7go71QMr5hDrs,10599
 verifiers/v1/tasksets/lean/taskset.py,sha256=XgUqX0uJvHC8sKFhbK92iu6nUGcPuXCUpI3ZBhDcb7c,9025
@@ -324,7 +325,7 @@ verifiers/v1/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuF
 verifiers/v1/utils/aio.py,sha256=0yNFOqB2oO6Qktc6NI_ZSAGuo1aHsIK_wuT1BtmoZ3I,1532
 verifiers/v1/utils/artifacts.py,sha256=vOzbPSuMirUecyylrhvLZ_vAcTd6JTJBztGPDQ0ANNk,6706
 verifiers/v1/utils/compile.py,sha256=Oh4hDVRNsql_BSOMh4kDEuze9_ZF17Q7uzBhhU70UZA,5663
-verifiers/v1/utils/decorators.py,sha256=9CcCwgZlRjVGPqT5G1CFJUHP4OIMuBGkUJZkcI6dnbk,3865
+verifiers/v1/utils/decorators.py,sha256=DG_K3HMiWqRHsY1bA7FJD3uTHUVuLw6SUJV-PnNRifA,4538
 verifiers/v1/utils/format.py,sha256=NfQo9M5KMzWCZpQaet5YtsJx56vCKAZPfbjZZJLJhhM,2187
 verifiers/v1/utils/generic.py,sha256=2VIN9RapMO9pVkFHB1UKUHck1vlTJ1pjfad2NzxuuXA,2055
 verifiers/v1/utils/git.py,sha256=MTRinOvG4EddfjdahHkFZUihHmhJ37n5ROtgjzjE1NA,8366
@@ -334,12 +335,12 @@ verifiers/v1/utils/interrupt.py,sha256=F-KKhc5ndPJJfhd3SuMqyqhXhA32FhCRy5KWFJpEo
 verifiers/v1/utils/loaders.py,sha256=NSIBdNU3JVeyddL-dM4dJt6khCHyjCgzJuFtLrgiwD0,9989
 verifiers/v1/utils/logging.py,sha256=OcMHA6NsYux3oIzjPuI95rDWmFBHNcHDjZeNIhXTX-Y,2084
 verifiers/v1/utils/memory.py,sha256=ZkIvGk6uITAH5sKon65LifKPbvZr8mJ__PVc23FZpOQ,1835
-verifiers/v1/utils/platform.py,sha256=MWZmvyj0iyzNIO0xRhlkCXtFF1plq6EJi8rWlvrFtvI,11363
+verifiers/v1/utils/platform.py,sha256=56Ixmk1SER6q5LvzyYcA0hgzGpzhxKZL8Hiqp4c_XVc,11576
 verifiers/v1/utils/retries.py,sha256=Y2ZgrAjn-qNkRKZP_RVNL_7EK0iaRcZeCa404lpGYi4,5417
 verifiers/v1/utils/score.py,sha256=-R5Cog14r_tJ6Q_oOfGr5VPAm2CCUhofYZB6B9lm4wM,5787
 verifiers/v1/utils/version.py,sha256=-obEo_-l9-D8FLef4hYxncOe-uJpxrM1g2Hig_37Sgs,1607
-verifiers-0.2.2.dev73.dist-info/METADATA,sha256=DU0GJRGJc9sAtCc2vIZa5E-O-4ubaCtIXJiCZQHs2rc,4545
-verifiers-0.2.2.dev73.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
-verifiers-0.2.2.dev73.dist-info/entry_points.txt,sha256=dF82JUYEFslR1AOW8LZfuBWeZeQoyX4wCRrduklg8-I,551
-verifiers-0.2.2.dev73.dist-info/licenses/LICENSE,sha256=v0RrUsdV3IDoZhrRce297IXS3xMHNJ-_LdLpFAUWb9k,1072
-verifiers-0.2.2.dev73.dist-info/RECORD,,
+verifiers-0.2.2.dev75.dist-info/METADATA,sha256=8ZNGDY1kLfAvLaXC5voMOQeGRtvHeflfC3eV0v34rWg,4545
+verifiers-0.2.2.dev75.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
+verifiers-0.2.2.dev75.dist-info/entry_points.txt,sha256=dF82JUYEFslR1AOW8LZfuBWeZeQoyX4wCRrduklg8-I,551
+verifiers-0.2.2.dev75.dist-info/licenses/LICENSE,sha256=v0RrUsdV3IDoZhrRce297IXS3xMHNJ-_LdLpFAUWb9k,1072
+verifiers-0.2.2.dev75.dist-info/RECORD,,