verifiers 0.3.2.dev195__tar.gz → 0.3.2.dev199__tar.gz

{verifiers-0.3.2.dev195 → verifiers-0.3.2.dev199}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.3.2.dev195
+Version: 0.3.2.dev199
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers
@@ -49,7 +49,7 @@ Requires-Dist: uvicorn>=0.52.0
 Provides-Extra: e2b
 Requires-Dist: e2b<3,>=2.35.0; extra == 'e2b'
 Provides-Extra: harbor
-Requires-Dist: harbor==0.21.0; (python_full_version >= '3.12') and extra == 'harbor'
+Requires-Dist: harbor==0.24.0; (python_full_version >= '3.12') and extra == 'harbor'
 Provides-Extra: modal
 Requires-Dist: modal>=1.5.4; extra == 'modal'
 Provides-Extra: openenv

{verifiers-0.3.2.dev195 → verifiers-0.3.2.dev199}/pyproject.toml

@@ -95,7 +95,7 @@ ta = [
 ]
 harbor = [
     # Harbor currently publishes Python 3.12+ packages; keep core verifiers installable on 3.11.
-    "harbor==0.21.0 ; python_full_version >= '3.12'",
+    "harbor==0.24.0 ; python_full_version >= '3.12'",
 ]
 modal = [
     "modal>=1.5.4",
@@ -143,7 +143,7 @@ web-search-interception = { path = "environments/web_search_interception", edita
 # Bounded cutoffs for the explicitly requested tool upgrades.
 # Full UTC timestamps: bare dates resolve to local-tz midnight and churn the
 # lockfile across timezones.
-harbor = "2026-08-11T00:00:00Z"
+harbor = "2026-10-08T00:00:00Z"
 ruff = "2026-07-28T00:00:00Z"
 ty = "2026-07-28T00:00:00Z"
 # PrimeIntellect-published on PyPI (trusted publisher)

{verifiers-0.3.2.dev195 → verifiers-0.3.2.dev199}/verifiers/v1/tasksets/harbor/taskset.py

@@ -10,9 +10,9 @@ grades that box (see ``env.py``). Either way the score lands in
 A pullable ``[environment].docker_image`` becomes ``TaskData.image``. Verifiers does
 not build Dockerfile-only environments, so those are rejected unless ``ignore_dockerfile``
 deliberately uses the harness runtime image. Tasks without an environment also use that
-image unless ``require_image`` is set. The same rule applies to a declared
-``[verifier.environment]``: it needs a pullable ``docker_image``, since Harbor would
-otherwise build the verifier image from ``tests/Dockerfile``.
+image unless ``require_image`` is set. Separate verifiers prefer their own image
+or tests build definition, then fall back to the agent environment. Unsupported verifier builds are rejected unless
+``ignore_dockerfile`` explicitly selects the fallback image.
 """
 
 import asyncio
@@ -28,7 +28,6 @@ import time
 from collections.abc import Iterator
 from functools import lru_cache
 from pathlib import Path, PurePosixPath
-from typing import Annotated
 
 from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError
 
@@ -50,12 +49,18 @@ HARBOR_INSTALL_HINT = "uv sync --python 3.12 --extra harbor"
 REWARD_JSON = "/logs/verifier/reward.json"
 MAX_REWARD_BYTES = 1024 * 1024
 REWARD_JSON_ADAPTER = TypeAdapter(
-    float | Annotated[dict[str, float], Field(min_length=1)],
+    dict[str, float],
     config=ConfigDict(strict=True, allow_inf_nan=False),
 )
 
+REWARD_SCALAR_ADAPTER = TypeAdapter(
+    float, config=ConfigDict(strict=True, allow_inf_nan=False)
+)
+
 
 class HarborTaskConfig(TaskConfig):
+    reward_key: str = Field("reward", min_length=1)
+    """Named verifier value used as the training reward; other values remain metrics."""
     mcp_servers: list[dict] = Field(default_factory=list)
     """Task-declared connections, bound from HarborData during construction."""
 
@@ -380,44 +385,57 @@ class HarborTask(Task[HarborData, State, HarborTaskConfig]):
             ["bash", "/tests/test.sh"], resolve_env(self.data.verifier_env)
         )
         scores = await self.read_reward_json(runtime)
-        if scores is not None:
-            if isinstance(scores, dict) and "reward" in scores:
-                trace.record_metrics(
-                    {key: value for key, value in scores.items() if key != "reward"}
-                )
-                return {"reward": scores["reward"]}
-            return scores
-        try:
-            reward = (
-                (
-                    await runtime.read(
-                        "/logs/verifier/reward.txt", max_bytes=MAX_REWARD_BYTES
+        if scores is None:
+            try:
+                value = (
+                    (
+                        await runtime.read(
+                            "/logs/verifier/reward.txt", max_bytes=MAX_REWARD_BYTES
+                        )
                     )
+                    .decode()
+                    .strip()
                 )
-                .decode()
-                .strip()
-            )
-            return REWARD_JSON_ADAPTER.validate_python(float(reward))
-        except (SandboxError, OSError, ValueError) as exc:
+                scores = {"reward": REWARD_SCALAR_ADAPTER.validate_python(float(value))}
+            except (SandboxError, OSError, ValueError) as exc:
+                raise TaskError(
+                    "Harbor verifier produced no reward.json and no valid reward.txt "
+                    f"(exit {result.exit_code}): "
+                    f"{(result.stderr or result.stdout).strip()[-500:]}"
+                ) from exc
+        key = self.config.reward_key
+        if key not in scores:
             raise TaskError(
-                "Harbor verifier produced no valid reward.json or reward.txt "
-                f"(exit {result.exit_code}): "
-                f"{(result.stderr or result.stdout).strip()[-500:]}"
-            ) from exc
+                f"Harbor verifier reward key {key!r} is missing; "
+                f"available keys: {sorted(scores)}. "
+                "Set task.reward_key to select a named verifier reward."
+            )
+        trace.record_metrics(
+            {name: value for name, value in scores.items() if name != key}
+        )
+        return {key: scores[key]}
 
-    async def read_reward_json(
-        self, runtime: Runtime
-    ) -> float | dict[str, float] | None:
-        """Read Harbor's scalar or keyed JSON reward, if it is valid.
+    async def read_reward_json(self, runtime: Runtime) -> dict[str, float] | None:
+        """JSON takes precedence whenever present; invalid or unreadable JSON fails.
 
-        Bounded: this is a grading input, and nothing guarantees its size.
+        Only absence permits text fallback. Reads remain bounded at the source.
         """
+        exists = await runtime.run(
+            ["sh", "-c", 'test -e "$1" || test -L "$1"', "vf-reward", REWARD_JSON], {}
+        )
+        if exists.exit_code == 1:
+            return None
+        if exists.exit_code:
+            raise TaskError(f"Cannot inspect Harbor reward file {REWARD_JSON}")
         try:
             return REWARD_JSON_ADAPTER.validate_json(
                 await runtime.read(REWARD_JSON, max_bytes=MAX_REWARD_BYTES)
             )
-        except (SandboxError, OSError, ValidationError):
-            return None
+        except (SandboxError, OSError, ValidationError) as exc:
+            raise TaskError(
+                "Harbor verifier produced invalid or unreadable reward.json; "
+                "expected an object of finite numeric rewards"
+            ) from exc
 
 
 def verifier_box_data(data: HarborData) -> HarborData:
@@ -442,7 +460,7 @@ def verifier_box_data(data: HarborData) -> HarborData:
             "workdir": data.workdir if fresh else verifier.workdir,
             "resources": data.resources if fresh else verifier.resources,
             "upload_environment": data.upload_environment
-            if fresh and data.verifier_image is None
+            if data.verifier_image is None
             else False,
             "env": dict(verifier.env),
             "healthcheck": verifier.healthcheck,
@@ -580,7 +598,7 @@ def resolve_image(
 
     ``None`` keeps the harness image for the solver, or the solver image for a
     separate verifier. A declared verifier environment without an image implies
-    a build from tests/Dockerfile, even if that file is absent.
+    a dedicated build only when the resolved definition contains one.
     """
     if image:
         return image
@@ -593,6 +611,8 @@ def resolve_image(
             return None
     section = "verifier.environment" if verifier else "environment"
     dockerfile = "tests/Dockerfile" if verifier else "environment/Dockerfile"
+    if verifier and not (task_dir / dockerfile).is_file():
+        dockerfile = "tests/docker-compose.yaml"
     if verifier or (task_dir / dockerfile).exists():
         if ignore_dockerfile:
             if verifier:
@@ -624,9 +644,15 @@ def parse_task(task_dir: Path, idx: int, harbor_config: HarborConfig) -> HarborD
     from harbor.environments.definition import should_upload_environment_dir
     from harbor.models.task.config import NetworkMode
     from harbor.models.task.task import Task as HarborModelTask
+    from harbor.models.task.verifier_mode import resolve_verifier_environment_definition
 
     harbor_task = HarborModelTask(task_dir)
     parsed = harbor_task.config
+    if parsed.steps:
+        raise ValueError(
+            f"{task_dir.name}: multi-step Harbor tasks are not supported; "
+            "select single-step tasks with --taskset.tasks"
+        )
     artifacts, hooks, verifier = parse_verifier_extras(task_dir, parsed, harbor_config)
     environment = parsed.environment
     image = resolve_image(
@@ -635,15 +661,20 @@ def parse_task(task_dir: Path, idx: int, harbor_config: HarborConfig) -> HarborD
         harbor_config.require_image,
         harbor_config.ignore_dockerfile,
     )
+    definition = (
+        resolve_verifier_environment_definition(parsed, harbor_task.paths)
+        if verifier is not None
+        else None
+    )
     verifier_image = (
         resolve_image(
             task_dir,
-            parsed.verifier.environment.docker_image,
+            definition.config.docker_image,
             require_image=True,
             ignore_dockerfile=harbor_config.ignore_dockerfile,
             verifier=True,
         )
-        if verifier is not None and parsed.verifier.environment is not None
+        if definition is not None and definition.bundled_tests
         else None
     )
     environment_dir = task_dir / "environment"
@@ -784,14 +815,15 @@ def parse_verifier_environment(
     Harbor resolves `[verifier.environment]` if declared, else a deep copy of
     `[environment]` — so a mode-only `separate` lands on the task's own image and needs
     nothing but a second box. A declared environment is the case that can name a
-    different image, and the case that can name none at all: there Harbor builds
-    `tests/Dockerfile`, which verifiers never does.
+    different image or only override resources. Image selection is resolved separately
+    by Harbor: verifier image, tests build definition, then the agent environment.
     """
     from harbor.models.task.config import NetworkMode, TaskOS
+    from harbor.models.task.paths import TaskPaths
     from harbor.models.task.verifier_mode import (
         VerifierEnvironmentMode,
-        resolve_effective_verifier_env_config,
         resolve_task_verifier_mode,
+        resolve_verifier_environment_definition,
     )
 
     if resolve_task_verifier_mode(parsed) != VerifierEnvironmentMode.SEPARATE:
@@ -804,9 +836,10 @@ def parse_verifier_environment(
         )
         return None
 
-    environment = resolve_effective_verifier_env_config(parsed, None)
-    if environment is None:  # unreachable while the mode is SEPARATE
+    definition = resolve_verifier_environment_definition(parsed, TaskPaths(task_dir))
+    if definition is None:  # unreachable while the mode is SEPARATE
         raise ValueError(f"{task_dir.name}: separate verifier resolved no environment")
+    environment = definition.config
     declared = parsed.verifier.environment is not None
     unsupported = [field for field in ("tpu",) if getattr(environment, field, None)]
     if environment.os != TaskOS.LINUX or unsupported:
@@ -866,10 +899,7 @@ class HarborTaskset(Taskset[HarborTask, HarborConfig]):
         task_dirs = [
             toml_path.parent
             for toml_path in sorted(root.rglob("task.toml"))
-            if (toml_path.parent / "instruction.md").is_file()
-            and (
-                self.config.tasks is None or toml_path.parent.name in self.config.tasks
-            )
+            if (self.config.tasks is None or toml_path.parent.name in self.config.tasks)
         ]
         if not task_dirs:
             raise ValueError(f"no harbor tasks found in {root}")