verifiers 0.2.2.dev40__py3-none-any.whl → 0.2.2.dev42__py3-none-any.whl

verifiers/v1/harness.py

@@ -9,7 +9,7 @@ from typing import TYPE_CHECKING, ClassVar, Generic, TypeVar
 from verifiers.v1.clients import ModelContext
 from verifiers.v1.configs.harness import HarnessConfig
 from verifiers.v1.decorators import discover_decorated, invoke_all
-from verifiers.v1.errors import HarnessError, boundary
+from verifiers.v1.errors import HarnessError, SandboxError, boundary
 from verifiers.v1.runtimes import ProgramResult, Runtime
 from verifiers.v1.task import TaskData
 from verifiers.v1.types import Messages
@@ -140,6 +140,11 @@ class Harness(ABC, Generic[ConfigT]):
         if result.exit_code != 0:
             # The real cause is at the END of a traceback, so keep the tail.
             detail = (result.stderr or result.stdout).strip()[-2000:] or "<no output>"
+            if not await runtime.alive():
+                raise SandboxError(
+                    f"runtime died under harness {self.config.id!r} "
+                    f"(exit {result.exit_code}): {detail}"
+                )
             raise HarnessError(
                 f"harness {self.config.id!r} exited {result.exit_code}: {detail}"
             )

verifiers/v1/harnesses/terminus_2/harness.py

@@ -13,7 +13,7 @@ logger = logging.getLogger(__name__)
 
 
 class Terminus2HarnessConfig(HarnessConfig):
-    version: str = "0.14.0"
+    version: str = "0.20.0"
     """Harbor release to install, pinned for reproducibility."""
 
 

verifiers/v1/runtimes/base.py

@@ -197,6 +197,16 @@ class Runtime(ABC):
     async def run(self, argv: list[str], env: dict[str, str]) -> ProgramResult:
         pass
 
+    async def alive(self) -> bool:
+        """Whether the box still executes anything. Not every runtime raises when
+        the box is gone — some surface it as `exec`'s own non-zero result,
+        indistinguishable from the command failing. One probe on the failure path
+        tells the two apart before we blame anyone."""
+        try:
+            return (await self.run(["true"], {})).exit_code == 0
+        except Exception:  # noqa: BLE001 - failing to exec at all means the box is gone
+            return False
+
     async def run_program(self, argv: list[str], env: dict[str, str]) -> ProgramResult:
         """Run the harness's MAIN program — the rollout itself (a possibly long-lived, stateful,
         agentic run) — as opposed to the short idempotent infra ops (write / mv / install /

verifiers/v1/tasksets/harbor/taskset.py

@@ -17,7 +17,6 @@ import subprocess
 import sys
 import tarfile
 import tempfile
-import tomllib
 from collections.abc import Iterator
 from functools import lru_cache
 from pathlib import Path
@@ -214,7 +213,7 @@ def dataset_dir(config: HarborConfig) -> Path:
 
 def resolve_image(
     task_dir: Path,
-    config: dict,
+    image: str | None,
     require_image: bool,
     ignore_dockerfile: bool = False,
 ) -> str | None:
@@ -224,9 +223,8 @@ def resolve_image(
     fallback for tasks with no environment, but would score a Dockerfile task in
     the wrong environment unless the user explicitly opts in.
     """
-    declared = config.get("environment", {}).get("docker_image")
-    if declared:
-        return declared
+    if image:
+        return image
     if (task_dir / "environment" / "Dockerfile").exists():
         if ignore_dockerfile:
             return None
@@ -243,67 +241,49 @@ def resolve_image(
     return None
 
 
-def size_to_mb(size: str | float) -> float:
-    """A Harbor size in MB, from either schema: current integer-MB fields or the
-    legacy schema-1.0 size strings ("8G", "512M", "64K")."""
-    if not isinstance(size, str):
-        return float(size)
-    scale = {"G": 1024.0, "M": 1.0, "K": 1 / 1024}.get(size.strip().upper()[-1:])
-    if scale is None:
-        raise ValueError(
-            f"invalid Harbor size {size!r}: expected a number of MB or a "
-            "'<number>[G|M|K]' string"
-        )
-    return float(size.strip()[:-1]) * scale
-
-
-def parse_resources(env: dict, multiplier: float = 1.0) -> TaskResources:
-    # Harbor's current schema reports memory/storage as integer-MB fields; the
-    # legacy schema 1.0 used size strings under `memory`/`storage`, which Harbor
-    # still migrates (datasets like senior-swe-bench are authored against it).
-    # TaskResources stores GB. A zero GPU count means no GPU request rather than
-    # the string "0".
-    memory = env.get("memory_mb") or env.get("memory")
-    disk = env.get("storage_mb") or env.get("storage")
-    return TaskResources(
-        cpu=env["cpus"] * multiplier if env.get("cpus") else None,
-        memory=size_to_mb(memory) / 1024 * multiplier if memory else None,
-        gpu=str(env["gpus"]) if env.get("gpus") else None,
-        disk=size_to_mb(disk) / 1024 * multiplier if disk else None,
-    )
-
-
 def parse_task(task_dir: Path, idx: int, harbor_config: HarborConfig) -> HarborData:
-    # Harbor is optional, so importing its schema is deferred until a Harbor task loads.
+    # Harbor is optional, so imports stay deferred until a Harbor task loads.
     from harbor.models.task.config import NetworkMode
-    from harbor.models.task.config import TaskConfig as HarborTaskConfig
-
-    config = tomllib.loads((task_dir / "task.toml").read_text())
-    parsed = HarborTaskConfig.model_validate(config)
-    network = (
-        parsed.agent.explicit_phase_policy() or parsed.environment.resolve_baseline()
+    from harbor.models.task.task import Task as HarborModelTask
+
+    harbor_task = HarborModelTask(task_dir)
+    parsed = harbor_task.config
+    environment = parsed.environment
+    network = parsed.agent.explicit_phase_policy() or environment.resolve_baseline()
+    task, meta = parsed.task, parsed.metadata
+    authors = (
+        [Author(name=author.name, email=author.email) for author in task.authors]
+        if task
+        else []
     )
-    task, meta = config.get("task", {}), config.get("metadata", {})
-    authors = [Author(**a) for a in task.get("authors", [])]
     # Older registry entries stored one author in [metadata].
     if not authors and meta.get("author_name"):
         authors = [Author(name=meta["author_name"], email=meta.get("author_email"))]
     if harbor_config.ignore_timeouts:
         harness_timeout = scoring_timeout = None
     else:
-        harness_timeout = config.get("agent", {}).get("timeout_sec")
-        scoring_timeout = config.get("verifier", {}).get("timeout_sec")
+        harness_timeout = (
+            parsed.agent.timeout_sec
+            if "timeout_sec" in parsed.agent.model_fields_set
+            else None
+        )
+        scoring_timeout = (
+            parsed.verifier.timeout_sec
+            if "timeout_sec" in parsed.verifier.model_fields_set
+            else None
+        )
     return HarborData(
         idx=idx,
-        name=task.get("name") or task_dir.name,
-        description=task.get("description"),
-        prompt=(task_dir / "instruction.md").read_text().strip(),
+        name=harbor_task.name,
+        description=task.description if task else None,
+        prompt=harbor_task.instruction.strip(),
         image=resolve_image(
             task_dir,
-            config,
+            environment.docker_image,
             harbor_config.require_image,
             harbor_config.ignore_dockerfile,
         ),
+        workdir=environment.workdir,
         network_allow=(
             ["*"]
             if network.network_mode == NetworkMode.PUBLIC
@@ -317,16 +297,25 @@ def parse_task(task_dir: Path, idx: int, harbor_config: HarborConfig) -> HarborD
             if scoring_timeout is not None
             else None,
         ),
-        resources=parse_resources(
-            config.get("environment", {}), harbor_config.resource_multiplier
+        resources=TaskResources(
+            cpu=environment.cpus * harbor_config.resource_multiplier
+            if environment.cpus
+            else None,
+            memory=environment.memory_mb / 1024 * harbor_config.resource_multiplier
+            if environment.memory_mb
+            else None,
+            gpu=str(environment.gpus) if environment.gpus else None,
+            disk=environment.storage_mb / 1024 * harbor_config.resource_multiplier
+            if environment.storage_mb
+            else None,
         ),
-        keywords=task.get("keywords", []),
+        keywords=task.keywords if task else [],
         authors=authors,
         difficulty=meta.get("difficulty"),
         category=meta.get("category"),
         tags=meta.get("tags", []),
         task_dir=str(task_dir),
-        verifier_env=config.get("verifier", {}).get("env", {}),
+        verifier_env=parsed.verifier.env,
     )
 
 

{verifiers-0.2.2.dev40.dist-info → verifiers-0.2.2.dev42.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.2.dev40
+Version: 0.2.2.dev42
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers
@@ -51,7 +51,7 @@ Provides-Extra: browser
 Requires-Dist: python-dotenv>=1.0.0; extra == 'browser'
 Requires-Dist: stagehand>=3.4.0; extra == 'browser'
 Provides-Extra: harbor
-Requires-Dist: harbor==0.14.0; (python_version >= '3.12') and extra == 'harbor'
+Requires-Dist: harbor==0.20.0; (python_version >= '3.12') and extra == 'harbor'
 Provides-Extra: modal
 Requires-Dist: modal>=1.4.0; extra == 'modal'
 Provides-Extra: notebook

{verifiers-0.2.2.dev40.dist-info → verifiers-0.2.2.dev42.dist-info}/RECORD

@@ -174,7 +174,7 @@ verifiers/v1/env.py,sha256=QFwhD1NKbEUWGf0l6G2ttK4JLbhUt2MUCDciWaaJ_1s,17834
 verifiers/v1/episode.py,sha256=KWyM9ovTphEIo2260BYu2HvtnOOTxjLRDPrF_jTCiuA,2497
 verifiers/v1/errors.py,sha256=Pj5Om8x1fP3TDPQQn6nUCoSPoZRsy2JeBz8pXhpPrDY,6883
 verifiers/v1/graph.py,sha256=-j8MNdIv1r5ifajv5ggnGPsfBNB74IO3WKLcrpYh9hg,29102
-verifiers/v1/harness.py,sha256=oL_CyPXw2ScCXfM-E0sk-7_y4PqvjxC8gq2jR2jyWUs,10709
+verifiers/v1/harness.py,sha256=cDiDuhp29aqL4CxiqSZz-IT0QlOcVlye_vTTHZblWo8,10948
 verifiers/v1/judge.py,sha256=VnTi9iSfCJMCccYWc1qCTcT7iD3AcomS2cXfHwh1HVw,9423
 verifiers/v1/legacy.py,sha256=kGfx4CEmk17VEqwX6JfUBbjQRMqJ0tL4kkA2kqGypN8,22305
 verifiers/v1/loaders.py,sha256=TnNKY2msVo3p7m-5wjw376PYH-zTu3TLixXRst0VOiM,9985
@@ -272,7 +272,7 @@ verifiers/v1/harnesses/pool/harness.py,sha256=6DlOA2W3s-NrxdQ9GaMmZg7mSNJGz2GC21
 verifiers/v1/harnesses/rlm/__init__.py,sha256=hwx51xTdEWzzYgMj-p4ETNcptoub3IPMDbEdJ4uB8Ns,122
 verifiers/v1/harnesses/rlm/harness.py,sha256=LRllWSQjpoq-IrOWVCp1sNvBgivqZYz9pVUsarZBQfA,7208
 verifiers/v1/harnesses/terminus_2/__init__.py,sha256=l1pwfg1HyVDEqT9efdnFWcXqaZwtKwe7TrplkyxZSA0,166
-verifiers/v1/harnesses/terminus_2/harness.py,sha256=-kDpMii6FBX6RfN7iHGZ8p6ZwMp94OwDhocKIWxEg0E,3027
+verifiers/v1/harnesses/terminus_2/harness.py,sha256=h-W2Yc4juKcg9YuGp7sZtJghTdu-S6pq4Wf5cP0VBeE,3027
 verifiers/v1/harnesses/terminus_2/program.py,sha256=XWflwULPfVUyrwAldXLRVOoUWoWW9bYNsaygWef0Sfg,2668
 verifiers/v1/interception/__init__.py,sha256=g2mLlps-NSlZjL3JJv76QBwVwMyMUaNt_9KsNrlgAAk,2980
 verifiers/v1/interception/base.py,sha256=WUsPqIkC9I3xnQCCjDCoX7jiJUiG_qt97oJrTKEgWY4,2698
@@ -292,7 +292,7 @@ verifiers/v1/mcp/launch.py,sha256=lRtcl98BO7imC44bsxO2yljwRQR_KCvJ6P-bd_hS5L8,18
 verifiers/v1/mcp/server.py,sha256=uW8qddXQbtECxIPCU-lkOlFIv3VKWoFAeZFNhwPoyWo,10985
 verifiers/v1/mcp/toolset.py,sha256=tSPXgTNREZyaeCnoFVnDfXxFAj_pStZMPUIgyPrY_Dk,996
 verifiers/v1/runtimes/__init__.py,sha256=TqNtJVahzQuLS4fgCVeT9xdOzTkp0hRAPOkl7eFVDUg,2011
-verifiers/v1/runtimes/base.py,sha256=hv1McFrIZgymLwMLNYPAlsR4At0uz60haVo5eXrFbNU,14196
+verifiers/v1/runtimes/base.py,sha256=sTCiLo51yAU_T-Voc3BTH-FuzyEM0cGY05VzFE4DmYc,14720
 verifiers/v1/runtimes/limiters.py,sha256=ZjjYUFEt20yH8ggL5uU9w9U500AOB44qSUTQ7bYyh0c,2730
 verifiers/v1/runtimes/modal.py,sha256=FnocoJd9lQVQS3V3CoBon7KpKXm1bKWWeU2FAkq2bKU,8825
 verifiers/v1/runtimes/prime.py,sha256=wldpcZnKZHNRldpoubeMgHdl-ZlKWOyeulFu4nHNKwc,15110
@@ -306,7 +306,7 @@ verifiers/v1/serve/server.py,sha256=gTrSZYSG2DH-KpAhLMN2L1oaMC6oJrqtguNrDfgHQFU,
 verifiers/v1/serve/types.py,sha256=AHFbt8C-Rzlee34RDPLQ3lPiYDLkUm5eFMnRQINiO6s,3016
 verifiers/v1/tasksets/__init__.py,sha256=b62O4WDNbGfJH_4NjYmznn5otWovdfvUgJZyoIZyLh8,596
 verifiers/v1/tasksets/harbor/__init__.py,sha256=wwVGA8N7E2Q2qA5WGHuLRrn4DgG5oaRHrCS2RW3NXVY,195
-verifiers/v1/tasksets/harbor/taskset.py,sha256=zRztBPzsZnuMa7lCDBDvaAyB5fQ2jRRUDHvrKwujai0,14731
+verifiers/v1/tasksets/harbor/taskset.py,sha256=aPgdVExCtt09t2YJrsK4UmlpQwAp5-htXa7arC1bX68,13877
 verifiers/v1/tasksets/lean/__init__.py,sha256=oyv-GbJQBvCyGwo7Ysxm2NjZW-aX4j1uDoHU0f-E7xQ,767
 verifiers/v1/tasksets/lean/scoring.py,sha256=sfzsT6MUK0zdMgAQ57_P_0CBaPHy3M7go71QMr5hDrs,10599
 verifiers/v1/tasksets/lean/taskset.py,sha256=0KujQgx2fTL_DyWSQidebKoD94U8oJp82PXoco6-3B4,9065
@@ -327,8 +327,8 @@ verifiers/v1/utils/logging.py,sha256=OcMHA6NsYux3oIzjPuI95rDWmFBHNcHDjZeNIhXTX-Y
 verifiers/v1/utils/memory.py,sha256=ZkIvGk6uITAH5sKon65LifKPbvZr8mJ__PVc23FZpOQ,1835
 verifiers/v1/utils/sampling.py,sha256=JczGzBn6s3wsIrSY2Hy3hE8m6NreNgsNzhSjDikQqX0,1037
 verifiers/v1/utils/version.py,sha256=-obEo_-l9-D8FLef4hYxncOe-uJpxrM1g2Hig_37Sgs,1607
-verifiers-0.2.2.dev40.dist-info/METADATA,sha256=7FRJgVM3XdGU4JTZSsGsxxaKndzkW26q4EFyz9NsMhE,4540
-verifiers-0.2.2.dev40.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
-verifiers-0.2.2.dev40.dist-info/entry_points.txt,sha256=dF82JUYEFslR1AOW8LZfuBWeZeQoyX4wCRrduklg8-I,551
-verifiers-0.2.2.dev40.dist-info/licenses/LICENSE,sha256=v0RrUsdV3IDoZhrRce297IXS3xMHNJ-_LdLpFAUWb9k,1072
-verifiers-0.2.2.dev40.dist-info/RECORD,,
+verifiers-0.2.2.dev42.dist-info/METADATA,sha256=Sfv8yHjesqeUSatEPoWH_rvAosXXPvZ2xczF2O7_mzI,4540
+verifiers-0.2.2.dev42.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
+verifiers-0.2.2.dev42.dist-info/entry_points.txt,sha256=dF82JUYEFslR1AOW8LZfuBWeZeQoyX4wCRrduklg8-I,551
+verifiers-0.2.2.dev42.dist-info/licenses/LICENSE,sha256=v0RrUsdV3IDoZhrRce297IXS3xMHNJ-_LdLpFAUWb9k,1072
+verifiers-0.2.2.dev42.dist-info/RECORD,,