verifiers 0.2.2.dev70__py3-none-any.whl → 0.2.2.dev72__py3-none-any.whl

verifiers/v1/tasksets/harbor/taskset.py

@@ -2,7 +2,7 @@
 
 The Harbor CLI downloads and caches each task directory. Its verifier runs in the
 same runtime the harness edited, then writes the score to
-``/logs/verifier/reward.txt``.
+``/logs/verifier/reward.json`` or the legacy ``reward.txt``.
 
 A pullable ``[environment].docker_image`` becomes ``TaskData.image``. Verifiers does
 not build Dockerfile-only environments, so those are rejected unless ``ignore_dockerfile``
@@ -21,8 +21,9 @@ import tempfile
 from collections.abc import Iterator
 from functools import lru_cache
 from pathlib import Path
+from typing import Annotated
 
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError
 
 from verifiers.v1.configs.taskset import TasksetConfig
 from verifiers.v1.errors import SandboxError
@@ -35,6 +36,11 @@ from verifiers.v1.utils.decorators import reward
 
 CACHE = Path.home() / ".cache" / "harbor"
 HARBOR_INSTALL_HINT = "uv sync --python 3.12 --extra harbor"
+REWARD_JSON = "/logs/verifier/reward.json"
+REWARD_JSON_ADAPTER = TypeAdapter(
+    float | Annotated[dict[str, float], Field(min_length=1)],
+    config=ConfigDict(strict=True, allow_inf_nan=False),
+)
 
 
 class HarborConfig(TasksetConfig):
@@ -140,7 +146,7 @@ class HarborTask(Task[HarborData]):
         trace.state.artifacts = await collect(runtime, self.data.artifacts)
 
     @reward(weight=1.0)
-    async def solved(self, runtime: Runtime) -> float:
+    async def solved(self, runtime: Runtime, trace: Trace) -> float | dict[str, float]:
         await runtime.write(
             "/tmp/tests.tgz", make_tar(Path(self.data.task_dir) / "tests")
         )
@@ -153,14 +159,37 @@ class HarborTask(Task[HarborData]):
             {},
         )
         await runtime.run(
-            ["sh", "-c", "cd /tests && bash test.sh"], verifier_env(self.data)
+            [
+                "sh",
+                "-c",
+                (
+                    "rm -f /logs/verifier/reward.json /logs/verifier/reward.txt"
+                    " && cd /tests && bash test.sh"
+                ),
+            ],
+            verifier_env(self.data),
         )
+        scores = await self._reward_json(runtime)
+        if scores is not None:
+            if isinstance(scores, dict) and "reward" in scores:
+                trace.record_metrics(
+                    {key: value for key, value in scores.items() if key != "reward"}
+                )
+                return {"reward": scores["reward"]}
+            return scores
         try:
             reward = (await runtime.read("/logs/verifier/reward.txt")).decode().strip()
             return float(reward or 0)
         except (SandboxError, OSError, ValueError):
             return 0.0
 
+    async def _reward_json(self, runtime: Runtime) -> float | dict[str, float] | None:
+        """Read Harbor's scalar or keyed JSON reward, if it is valid."""
+        try:
+            return REWARD_JSON_ADAPTER.validate_json(await runtime.read(REWARD_JSON))
+        except (SandboxError, OSError, ValidationError):
+            return None
+
 
 def harbor_cli() -> str:
     scripts_dir = Path(sys.executable).parent

{verifiers-0.2.2.dev70.dist-info → verifiers-0.2.2.dev72.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.2.dev70
+Version: 0.2.2.dev72
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.2.2.dev70.dist-info → verifiers-0.2.2.dev72.dist-info}/RECORD

@@ -311,7 +311,7 @@ verifiers/v1/serve/server.py,sha256=o2GjMgnNApyN9A2i_7SVL3ui75NTBknQlobfUOVTfdQ,
 verifiers/v1/serve/types.py,sha256=y8Cf9wKOZRKDXcThFWVygTFHITJofOinGiIzwjQGCUE,2674
 verifiers/v1/tasksets/__init__.py,sha256=b62O4WDNbGfJH_4NjYmznn5otWovdfvUgJZyoIZyLh8,596
 verifiers/v1/tasksets/harbor/__init__.py,sha256=wwVGA8N7E2Q2qA5WGHuLRrn4DgG5oaRHrCS2RW3NXVY,195
-verifiers/v1/tasksets/harbor/taskset.py,sha256=7cUb7um8FUL2vHhvdSVBN04IwFMzYDjdaAEuMqiPftM,18326
+verifiers/v1/tasksets/harbor/taskset.py,sha256=9rfWTg0s3zPAEppZnjh7zNyi2EbM-qORK2lyWQZQL7s,19565
 verifiers/v1/tasksets/lean/__init__.py,sha256=oyv-GbJQBvCyGwo7Ysxm2NjZW-aX4j1uDoHU0f-E7xQ,767
 verifiers/v1/tasksets/lean/scoring.py,sha256=sfzsT6MUK0zdMgAQ57_P_0CBaPHy3M7go71QMr5hDrs,10599
 verifiers/v1/tasksets/lean/taskset.py,sha256=XgUqX0uJvHC8sKFhbK92iu6nUGcPuXCUpI3ZBhDcb7c,9025
@@ -337,8 +337,8 @@ verifiers/v1/utils/platform.py,sha256=MWZmvyj0iyzNIO0xRhlkCXtFF1plq6EJi8rWlvrFtv
 verifiers/v1/utils/retries.py,sha256=Y2ZgrAjn-qNkRKZP_RVNL_7EK0iaRcZeCa404lpGYi4,5417
 verifiers/v1/utils/score.py,sha256=-R5Cog14r_tJ6Q_oOfGr5VPAm2CCUhofYZB6B9lm4wM,5787
 verifiers/v1/utils/version.py,sha256=-obEo_-l9-D8FLef4hYxncOe-uJpxrM1g2Hig_37Sgs,1607
-verifiers-0.2.2.dev70.dist-info/METADATA,sha256=KIrPpfHWXuZ95thI2Vymh--9HQa429Q1V_YI0PMliYg,4545
-verifiers-0.2.2.dev70.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
-verifiers-0.2.2.dev70.dist-info/entry_points.txt,sha256=dF82JUYEFslR1AOW8LZfuBWeZeQoyX4wCRrduklg8-I,551
-verifiers-0.2.2.dev70.dist-info/licenses/LICENSE,sha256=v0RrUsdV3IDoZhrRce297IXS3xMHNJ-_LdLpFAUWb9k,1072
-verifiers-0.2.2.dev70.dist-info/RECORD,,
+verifiers-0.2.2.dev72.dist-info/METADATA,sha256=6Il792lPzYePRqxmMiC5qTR_-h5jLSRa2cu2spucfMQ,4545
+verifiers-0.2.2.dev72.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
+verifiers-0.2.2.dev72.dist-info/entry_points.txt,sha256=dF82JUYEFslR1AOW8LZfuBWeZeQoyX4wCRrduklg8-I,551
+verifiers-0.2.2.dev72.dist-info/licenses/LICENSE,sha256=v0RrUsdV3IDoZhrRce297IXS3xMHNJ-_LdLpFAUWb9k,1072
+verifiers-0.2.2.dev72.dist-info/RECORD,,