verifiers 0.2.2.dev40__tar.gz → 0.2.2.dev42__tar.gz

{verifiers-0.2.2.dev40 → verifiers-0.2.2.dev42}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.2.dev40
+Version: 0.2.2.dev42
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers
@@ -51,7 +51,7 @@ Provides-Extra: browser
 Requires-Dist: python-dotenv>=1.0.0; extra == 'browser'
 Requires-Dist: stagehand>=3.4.0; extra == 'browser'
 Provides-Extra: harbor
-Requires-Dist: harbor==0.14.0; (python_version >= '3.12') and extra == 'harbor'
+Requires-Dist: harbor==0.20.0; (python_version >= '3.12') and extra == 'harbor'
 Provides-Extra: modal
 Requires-Dist: modal>=1.4.0; extra == 'modal'
 Provides-Extra: notebook

{verifiers-0.2.2.dev40 → verifiers-0.2.2.dev42}/pyproject.toml

@@ -96,7 +96,7 @@ browser = [
 ]
 harbor = [
     # Harbor currently publishes Python 3.12+ packages; keep core verifiers installable on 3.11.
-    "harbor==0.14.0; python_version >= '3.12'",
+    "harbor==0.20.0; python_version >= '3.12'",
 ]
 modal = [
     "modal>=1.4.0",

{verifiers-0.2.2.dev40 → verifiers-0.2.2.dev42}/verifiers/v1/harness.py

@@ -9,7 +9,7 @@ from typing import TYPE_CHECKING, ClassVar, Generic, TypeVar
 from verifiers.v1.clients import ModelContext
 from verifiers.v1.configs.harness import HarnessConfig
 from verifiers.v1.decorators import discover_decorated, invoke_all
-from verifiers.v1.errors import HarnessError, boundary
+from verifiers.v1.errors import HarnessError, SandboxError, boundary
 from verifiers.v1.runtimes import ProgramResult, Runtime
 from verifiers.v1.task import TaskData
 from verifiers.v1.types import Messages
@@ -140,6 +140,11 @@ class Harness(ABC, Generic[ConfigT]):
         if result.exit_code != 0:
             # The real cause is at the END of a traceback, so keep the tail.
             detail = (result.stderr or result.stdout).strip()[-2000:] or "<no output>"
+            if not await runtime.alive():
+                raise SandboxError(
+                    f"runtime died under harness {self.config.id!r} "
+                    f"(exit {result.exit_code}): {detail}"
+                )
             raise HarnessError(
                 f"harness {self.config.id!r} exited {result.exit_code}: {detail}"
             )

{verifiers-0.2.2.dev40 → verifiers-0.2.2.dev42}/verifiers/v1/harnesses/terminus_2/harness.py

@@ -13,7 +13,7 @@ logger = logging.getLogger(__name__)
 
 
 class Terminus2HarnessConfig(HarnessConfig):
-    version: str = "0.14.0"
+    version: str = "0.20.0"
     """Harbor release to install, pinned for reproducibility."""
 
 

{verifiers-0.2.2.dev40 → verifiers-0.2.2.dev42}/verifiers/v1/runtimes/base.py

@@ -197,6 +197,16 @@ class Runtime(ABC):
     async def run(self, argv: list[str], env: dict[str, str]) -> ProgramResult:
         pass
 
+    async def alive(self) -> bool:
+        """Whether the box still executes anything. Not every runtime raises when
+        the box is gone — some surface it as `exec`'s own non-zero result,
+        indistinguishable from the command failing. One probe on the failure path
+        tells the two apart before we blame anyone."""
+        try:
+            return (await self.run(["true"], {})).exit_code == 0
+        except Exception:  # noqa: BLE001 - failing to exec at all means the box is gone
+            return False
+
     async def run_program(self, argv: list[str], env: dict[str, str]) -> ProgramResult:
         """Run the harness's MAIN program — the rollout itself (a possibly long-lived, stateful,
         agentic run) — as opposed to the short idempotent infra ops (write / mv / install /

{verifiers-0.2.2.dev40 → verifiers-0.2.2.dev42}/verifiers/v1/tasksets/harbor/taskset.py

@@ -17,7 +17,6 @@ import subprocess
 import sys
 import tarfile
 import tempfile
-import tomllib
 from collections.abc import Iterator
 from functools import lru_cache
 from pathlib import Path
@@ -214,7 +213,7 @@ def dataset_dir(config: HarborConfig) -> Path:
 
 def resolve_image(
     task_dir: Path,
-    config: dict,
+    image: str | None,
     require_image: bool,
     ignore_dockerfile: bool = False,
 ) -> str | None:
@@ -224,9 +223,8 @@ def resolve_image(
     fallback for tasks with no environment, but would score a Dockerfile task in
     the wrong environment unless the user explicitly opts in.
     """
-    declared = config.get("environment", {}).get("docker_image")
-    if declared:
-        return declared
+    if image:
+        return image
     if (task_dir / "environment" / "Dockerfile").exists():
         if ignore_dockerfile:
             return None
@@ -243,67 +241,49 @@ def resolve_image(
     return None
 
 
-def size_to_mb(size: str | float) -> float:
-    """A Harbor size in MB, from either schema: current integer-MB fields or the
-    legacy schema-1.0 size strings ("8G", "512M", "64K")."""
-    if not isinstance(size, str):
-        return float(size)
-    scale = {"G": 1024.0, "M": 1.0, "K": 1 / 1024}.get(size.strip().upper()[-1:])
-    if scale is None:
-        raise ValueError(
-            f"invalid Harbor size {size!r}: expected a number of MB or a "
-            "'<number>[G|M|K]' string"
-        )
-    return float(size.strip()[:-1]) * scale
-
-
-def parse_resources(env: dict, multiplier: float = 1.0) -> TaskResources:
-    # Harbor's current schema reports memory/storage as integer-MB fields; the
-    # legacy schema 1.0 used size strings under `memory`/`storage`, which Harbor
-    # still migrates (datasets like senior-swe-bench are authored against it).
-    # TaskResources stores GB. A zero GPU count means no GPU request rather than
-    # the string "0".
-    memory = env.get("memory_mb") or env.get("memory")
-    disk = env.get("storage_mb") or env.get("storage")
-    return TaskResources(
-        cpu=env["cpus"] * multiplier if env.get("cpus") else None,
-        memory=size_to_mb(memory) / 1024 * multiplier if memory else None,
-        gpu=str(env["gpus"]) if env.get("gpus") else None,
-        disk=size_to_mb(disk) / 1024 * multiplier if disk else None,
-    )
-
-
 def parse_task(task_dir: Path, idx: int, harbor_config: HarborConfig) -> HarborData:
-    # Harbor is optional, so importing its schema is deferred until a Harbor task loads.
+    # Harbor is optional, so imports stay deferred until a Harbor task loads.
     from harbor.models.task.config import NetworkMode
-    from harbor.models.task.config import TaskConfig as HarborTaskConfig
-
-    config = tomllib.loads((task_dir / "task.toml").read_text())
-    parsed = HarborTaskConfig.model_validate(config)
-    network = (
-        parsed.agent.explicit_phase_policy() or parsed.environment.resolve_baseline()
+    from harbor.models.task.task import Task as HarborModelTask
+
+    harbor_task = HarborModelTask(task_dir)
+    parsed = harbor_task.config
+    environment = parsed.environment
+    network = parsed.agent.explicit_phase_policy() or environment.resolve_baseline()
+    task, meta = parsed.task, parsed.metadata
+    authors = (
+        [Author(name=author.name, email=author.email) for author in task.authors]
+        if task
+        else []
     )
-    task, meta = config.get("task", {}), config.get("metadata", {})
-    authors = [Author(**a) for a in task.get("authors", [])]
     # Older registry entries stored one author in [metadata].
     if not authors and meta.get("author_name"):
         authors = [Author(name=meta["author_name"], email=meta.get("author_email"))]
     if harbor_config.ignore_timeouts:
         harness_timeout = scoring_timeout = None
     else:
-        harness_timeout = config.get("agent", {}).get("timeout_sec")
-        scoring_timeout = config.get("verifier", {}).get("timeout_sec")
+        harness_timeout = (
+            parsed.agent.timeout_sec
+            if "timeout_sec" in parsed.agent.model_fields_set
+            else None
+        )
+        scoring_timeout = (
+            parsed.verifier.timeout_sec
+            if "timeout_sec" in parsed.verifier.model_fields_set
+            else None
+        )
     return HarborData(
         idx=idx,
-        name=task.get("name") or task_dir.name,
-        description=task.get("description"),
-        prompt=(task_dir / "instruction.md").read_text().strip(),
+        name=harbor_task.name,
+        description=task.description if task else None,
+        prompt=harbor_task.instruction.strip(),
         image=resolve_image(
             task_dir,
-            config,
+            environment.docker_image,
             harbor_config.require_image,
             harbor_config.ignore_dockerfile,
         ),
+        workdir=environment.workdir,
         network_allow=(
             ["*"]
             if network.network_mode == NetworkMode.PUBLIC
@@ -317,16 +297,25 @@ def parse_task(task_dir: Path, idx: int, harbor_config: HarborConfig) -> HarborD
             if scoring_timeout is not None
             else None,
         ),
-        resources=parse_resources(
-            config.get("environment", {}), harbor_config.resource_multiplier
+        resources=TaskResources(
+            cpu=environment.cpus * harbor_config.resource_multiplier
+            if environment.cpus
+            else None,
+            memory=environment.memory_mb / 1024 * harbor_config.resource_multiplier
+            if environment.memory_mb
+            else None,
+            gpu=str(environment.gpus) if environment.gpus else None,
+            disk=environment.storage_mb / 1024 * harbor_config.resource_multiplier
+            if environment.storage_mb
+            else None,
         ),
-        keywords=task.get("keywords", []),
+        keywords=task.keywords if task else [],
         authors=authors,
         difficulty=meta.get("difficulty"),
         category=meta.get("category"),
         tags=meta.get("tags", []),
         task_dir=str(task_dir),
-        verifier_env=config.get("verifier", {}).get("env", {}),
+        verifier_env=parsed.verifier.env,
     )