verifiers 0.3.2.dev70__tar.gz → 0.3.2.dev71__tar.gz

{verifiers-0.3.2.dev70 → verifiers-0.3.2.dev71}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.3.2.dev70
+Version: 0.3.2.dev71
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.3.2.dev70 → verifiers-0.3.2.dev71}/verifiers/v1/tasksets/harbor/taskset.py

@@ -27,7 +27,7 @@ import tempfile
 import time
 from collections.abc import Iterator
 from functools import lru_cache
-from pathlib import Path
+from pathlib import Path, PurePosixPath
 from typing import Annotated
 
 from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError
@@ -250,31 +250,38 @@ class HarborTask(Task[HarborData]):
         if not self.scoring_deferred:
             trace.state.artifacts = await collect(runtime, self.data.artifacts)
 
-    async def stage_verifier(self, runtime: Runtime) -> None:
-        # An artifact under /tests must not replace the task package's verifier.
+    async def stage_verifier(self, trace: Trace, runtime: Runtime) -> None:
+        if any(
+            PurePosixPath(root).is_relative_to("/tests")
+            for root in trace.state.artifacts
+        ):
+            raise TaskError("Harbor artifacts cannot restore into /tests")
         await self.stage_tests(runtime, wipe=True)
         self.verifier_staged = True
 
     async def stage_tests(self, runtime: Runtime, wipe: bool = False) -> None:
-        """Put the task package's `tests/` in `/tests`, where `test.sh` expects it.
+        """Use a dedicated verifier image's tests, or stage the task package's tests.
 
         Raises rather than scoring stale state: a leftover reward file — planted by
         the agent or shipped in the image — must be gone before `test.sh` runs, so a
         removal that fails must not fall through to reading it.
-
-        `wipe` for a box we did not watch being built: a fresh container of the task's
-        image can ship its own `/tests`, and a leftover file there would be graded as
-        though it came from the package.
         """
-        await runtime.write(
-            "/tmp/tests.tgz", make_tar(Path(self.data.task_dir) / "tests")
-        )
-        stage = (
-            f"{'rm -rf /tests && ' if wipe else ''}"
+        # Harbor's dedicated verifier image owns the complete test suite and its
+        # dependencies. Mixing it with packaged tests can retain obsolete helpers.
+        stage = "test -f /tests/test.sh"
+        if self.data.verifier is None or self.data.verifier.image is None:
+            await runtime.write(
+                "/tmp/tests.tgz", make_tar(Path(self.data.task_dir) / "tests")
+            )
+            stage = (
+                f"{'rm -rf /tests && ' if wipe else ''}"
+                "mkdir -p /tests && tar -xzf /tmp/tests.tgz -C /tests"
+            )
+        command = (
             "rm -f /logs/verifier/reward.json /logs/verifier/reward.txt && "
-            "mkdir -p /logs/verifier /tests && tar -xzf /tmp/tests.tgz -C /tests"
+            f"mkdir -p /logs/verifier && {stage}"
         )
-        result = await runtime.run(["sh", "-c", stage], {})
+        result = await runtime.run(["sh", "-c", command], {})
         if result.exit_code:
             raise TaskError(
                 f"staging tests failed (exit {result.exit_code}): "