verifiers 0.3.2.dev116__py3-none-any.whl → 0.3.2.dev117__py3-none-any.whl

verifiers/v1/rollout.py

@@ -485,7 +485,9 @@ class Rollout:
                         )
                         if self._collect_artifacts and not trace.state.artifacts:
                             trace.state.artifacts = await collect(
-                                runtime, self.task.data.artifacts
+                                runtime,
+                                self.task.data.artifacts,
+                                max_bytes=self.task.data.artifact_max_bytes,
                             )
                 now = time.time()
                 trace.timing.finalize.end = now

verifiers/v1/runtimes/prime.py

@@ -1,8 +1,8 @@
 """Remote Prime VM sandbox runtime."""
 
 import asyncio
-import base64
 import contextlib
+import io
 import logging
 import math
 import shlex
@@ -329,19 +329,30 @@ class PrimeRuntime(Runtime):
     async def _read(self, path: str, max_bytes: int | None = None) -> bytes:
         if max_bytes is not None:
             try:
-                # VM execute_command uses bash and returns the complete output stream.
-                result = await self._client.execute_command(
+                # Stream binary output: execute_command buffers base64 text for the
+                # entire file. Bound the source read and the host buffer independently.
+                process = await self._client.open_process(
                     self.info.id,
-                    f"set -o pipefail; head -c {max_bytes} -- {shlex.quote(path)} | base64",
+                    f"head -c {max_bytes} -- {shlex.quote(path)}",
                     working_dir=self.config.workdir,
                     env=self.process_env({}),
-                    timeout=EFFECTIVELY_UNBOUNDED_SECONDS,
                 )
+                async with contextlib.aclosing(process):
+                    with io.BytesIO() as data:
+                        async for chunk in process.stdout:
+                            if data.tell() + len(chunk) > max_bytes:
+                                raise SandboxError(
+                                    "read stream exceeded its byte limit"
+                                )
+                            data.write(chunk)
+                        stderr = b""
+                        async for chunk in process.stderr:
+                            stderr = (stderr + chunk)[-500:]
+                        if await process.wait():
+                            raise SandboxError(stderr.decode(errors="replace").strip())
+                        return data.getvalue()
             except Exception as exc:
                 raise SandboxError(f"read {path!r}: {exc}") from exc
-            if result.exit_code:
-                raise SandboxError(f"read {path!r}: {result.stderr.strip()[-500:]}")
-            return base64.b64decode(result.stdout)
         # Avoid background-job log limits and base64 overhead by downloading binary data directly.
         # The temporary file is removed on every exit, and its byte read stays off the event loop.
         target = (

verifiers/v1/task.py

@@ -25,7 +25,7 @@ from verifiers.v1.configs.task import TaskConfig
 from verifiers.v1.errors import TaskError, boundary
 from verifiers.v1.state import StateT
 from verifiers.v1.types import Messages, content_text
-from verifiers.v1.utils.artifacts import Artifact
+from verifiers.v1.utils.artifacts import MAX_ARTIFACT_BYTES, Artifact
 from verifiers.v1.utils.decorators import (
     discover_decorated,
     invoke_all,
@@ -115,6 +115,8 @@ class TaskData(BaseModel):
     on top of the implicitly collected `/logs/artifacts/` convention dir. Declare
     runtime outputs that must cross that boundary. A declared path that is missing at
     collection time fails the rollout."""
+    artifact_max_bytes: int = Field(MAX_ARTIFACT_BYTES, gt=0)
+    """Total byte limit for collected artifact archives, including the convention dir."""
 
     timeout: TaskTimeout = TaskTimeout()
     resources: TaskResources = TaskResources()

verifiers/v1/tasksets/harbor/taskset.py

@@ -40,7 +40,7 @@ from verifiers.v1.state import State
 from verifiers.v1.task import Task, TaskData, TaskResources, TaskTimeout
 from verifiers.v1.taskset import Taskset
 from verifiers.v1.trace import Trace
-from verifiers.v1.utils.artifacts import Artifact, collect
+from verifiers.v1.utils.artifacts import MAX_ARTIFACT_BYTES, Artifact, collect
 from verifiers.v1.utils.decorators import reward
 
 logger = logging.getLogger(__name__)
@@ -61,6 +61,8 @@ class HarborTaskConfig(TaskConfig):
 
 
 class HarborConfig(TasksetConfig):
+    artifact_max_bytes: int = Field(MAX_ARTIFACT_BYTES, gt=0)
+    """Total byte limit for artifact archives transferred out of each solver runtime."""
     task: HarborTaskConfig = HarborTaskConfig()
     dataset: str = "harbor/hello-world"
     """A Harbor Hub package id ("org/name" or "org/name@ref"), where ref is a
@@ -272,7 +274,9 @@ class HarborTask(Task[HarborData, State, HarborTaskConfig]):
                     f"{hook.command}\n{detail}"
                 )
         if not self.scoring_deferred:
-            trace.state.artifacts = await collect(runtime, self.data.artifacts)
+            trace.state.artifacts = await collect(
+                runtime, self.data.artifacts, max_bytes=self.data.artifact_max_bytes
+            )
 
     async def stage_verifier(self, trace: Trace, runtime: Runtime) -> None:
         if any(
@@ -582,6 +586,7 @@ def parse_task(task_dir: Path, idx: int, harbor_config: HarborConfig) -> HarborD
         )
     return HarborData(
         idx=idx,
+        artifact_max_bytes=harbor_config.artifact_max_bytes,
         name=harbor_task.name,
         description=task.description if task else None,
         prompt=harbor_task.instruction.strip(),

verifiers/v1/utils/artifacts.py

@@ -21,7 +21,7 @@ ARTIFACTS_DIR = "/logs/artifacts"
 """Implicit artifact directory; tasks that write here need no declaration."""
 
 MAX_ARTIFACT_BYTES = 32 * 1024 * 1024
-"""Ceiling per collection. Sized for a delta, not a tree: the grading box boots from the
+"""Default ceiling per collection. Sized for a delta, not a tree: the grading box boots from the
 agent's image, so the repo is already there and only its output has to travel."""
 
 
@@ -35,7 +35,10 @@ class Artifact(BaseModel):
 
 
 async def collect(
-    runtime: Runtime, artifacts: list[Artifact] | None = None
+    runtime: Runtime,
+    artifacts: list[Artifact] | None = None,
+    *,
+    max_bytes: int = MAX_ARTIFACT_BYTES,
 ) -> dict[str, bytes | None]:
     """Tar the convention dir and every declared path out of `runtime`.
 
@@ -114,7 +117,7 @@ async def collect(
         )
         existence.extend(output.splitlines())
     collected: dict[str, bytes | None] = {}
-    budget = MAX_ARTIFACT_BYTES
+    budget = max_bytes
     for artifact, exists in zip(entries, existence, strict=True):
         source = artifact.source
         if exists != "1":

{verifiers-0.3.2.dev116.dist-info → verifiers-0.3.2.dev117.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.3.2.dev116
+Version: 0.3.2.dev117
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.3.2.dev116.dist-info → verifiers-0.3.2.dev117.dist-info}/RECORD

@@ -8,11 +8,11 @@ verifiers/v1/errors.py,sha256=uQySmCUrjDYFkBIii1vBFPiEcPFPQZ9EGpas_6HvcQI,5279
 verifiers/v1/graph.py,sha256=b3Z3lCnsALnAvY82pDnjhODFYakkeLV_hXgGOvbryrQ,40737
 verifiers/v1/harness.py,sha256=EmOmoen9ACsEgoUoS9-A7jpS2RyDg6ZlDbGo4APHU8E,15367
 verifiers/v1/judge.py,sha256=uqn-djkK2_7Q3kOjkDo6ipOvV_HOvuLa9y6AoDjRux8,9618
-verifiers/v1/rollout.py,sha256=oka3sNdv1EbE6LatFFRMJH-5ti6FcqFuD8xa7oVUyDA,24405
+verifiers/v1/rollout.py,sha256=yN6MoQ3ZiXoLUhg4MxKAmM1zDpukyqyGZHsc0tushfc,24515
 verifiers/v1/semantic.py,sha256=VQ_buw6Ya6QtRoh_fiYI5Avs5RmIQCfWH0H0bQc4aoc,4704
 verifiers/v1/session.py,sha256=7iarEDRH2epID_O8ai-1HwveXze_o8shnLw9pFMFX5o,18900
 verifiers/v1/state.py,sha256=EckF2bWp-vV4b1jYJ9sLI5xrfGuI5spIgYYwW926toI,595
-verifiers/v1/task.py,sha256=7iJLXQo7JKX5jG87atIwT_HZlSto_3a7yB3w__uQQQw,12427
+verifiers/v1/task.py,sha256=j7pX-MMZwixSIp4344V5KDI443VGRL4YEs8S6Ukwmr8,12599
 verifiers/v1/taskset.py,sha256=fp2E0IEhL_Ybj9cegZwljfTmW27p_30zTWHFKw_kXE4,4374
 verifiers/v1/trace.py,sha256=q88vHOr3ZwvhzXOneBx00ZQ82fry7UT4DCwidQnbk3Q,32120
 verifiers/v1/types.py,sha256=PH1SxRWXOjPCgQD9aZgM18_bECvFSNQ25gkCbulmTKU,9857
@@ -145,7 +145,7 @@ verifiers/v1/runtimes/base.py,sha256=IKmrJSJltWX9M1nPUZExKluPJM4N82VsnqQGFz0ECV8
 verifiers/v1/runtimes/container.py,sha256=puQD8X9acP6GefoCNEnq40POi-nXPD-729bUjTVShfM,9554
 verifiers/v1/runtimes/limiters.py,sha256=C6cWDD4pwyFwUY4QFke3oiMXwydSNIfSbJSIglUC2kE,3062
 verifiers/v1/runtimes/modal.py,sha256=rFbxdDBi_VIiDA67XyZg-AOsc_XkV8aQWSKnTUAgCmU,17506
-verifiers/v1/runtimes/prime.py,sha256=eeYGu60f8PSGtL6g0nJslYzrQLHo6d64JP73N3q3xTw,16982
+verifiers/v1/runtimes/prime.py,sha256=BXwDK-eurEzOKLY1ezuC0UYZdEoI0Y9Cfcd0IhKjMPc,17586
 verifiers/v1/runtimes/subprocess.py,sha256=QmpQ235_xrX6gowrr8557VgbydOXpLbfFq0KKGfJThk,8774
 verifiers/v1/runtimes/docker/__init__.py,sha256=2wceP-8V-w0dQBEeLtoJFfCZ135DxP2rcdcUBmSWiNU,18543
 verifiers/v1/runtimes/docker/egress.py,sha256=wtKWTL2_zFidxMR7lDRmfevQlOsl2Iyk88ebdO4Kqqw,24566
@@ -159,7 +159,7 @@ verifiers/v1/serve/types.py,sha256=Qw5IjiJhsZz4giSXUBi9dfInBnQZ5_xiUmgH2qp8jPI,1
 verifiers/v1/tasksets/__init__.py,sha256=5hTKSPLXSZTgKBbeKQI7FEp06c0ZsaQfuwvrvVFLgb8,712
 verifiers/v1/tasksets/harbor/__init__.py,sha256=JxThzHFYEkuSfQbVf3pEnjordgdfe4MU-QOsSSbyovU,326
 verifiers/v1/tasksets/harbor/env.py,sha256=ATMkfwmrE_2jpOOffbjLQ91feG7TOtefBGlpwWbRq8c,3604
-verifiers/v1/tasksets/harbor/taskset.py,sha256=3xCxtVVwQSpaLfoV1SaUoNQlpIGaTmwW6PvPyYh3yv4,33780
+verifiers/v1/tasksets/harbor/taskset.py,sha256=35ljhNapn1EOlaYFTuVZxAMVU0KDfmFxQOGOcUk2Ow0,34082
 verifiers/v1/tasksets/harbor/toolset.py,sha256=_7HXu-uuvqzhmJ1EX6c3KWLg0g0eRrLjCq1kBsZhIwo,3159
 verifiers/v1/tasksets/lean/__init__.py,sha256=oyv-GbJQBvCyGwo7Ysxm2NjZW-aX4j1uDoHU0f-E7xQ,767
 verifiers/v1/tasksets/lean/scoring.py,sha256=sfzsT6MUK0zdMgAQ57_P_0CBaPHy3M7go71QMr5hDrs,10599
@@ -175,7 +175,7 @@ verifiers/v1/tasksets/textarena/__init__.py,sha256=Os2OlBY_pSH0B1DP32RitumgSLpq2
 verifiers/v1/tasksets/textarena/taskset.py,sha256=9fa_unlFiFZyuN4I02rW3x4qyRnFWXqwTUZTloQefu0,4433
 verifiers/v1/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 verifiers/v1/utils/aio.py,sha256=0yNFOqB2oO6Qktc6NI_ZSAGuo1aHsIK_wuT1BtmoZ3I,1532
-verifiers/v1/utils/artifacts.py,sha256=PFrSkwAyvzHyT6JjlFeGOPIXDmS-uRajaN3am4HZ0TI,9263
+verifiers/v1/utils/artifacts.py,sha256=3so4KLKlrtF2sM_jMZUAyjTHWnfuUltGX1MrrLns-tI,9315
 verifiers/v1/utils/compile.py,sha256=5pxy31wx2Q-3pQjOO5B9SH6OT-wwb8r4F1LAl6HMeQA,5835
 verifiers/v1/utils/decorators.py,sha256=vhoN6YoKb15-Fkqmc9RaNlUIbt9WRvNK4zAn2bAxfS4,5367
 verifiers/v1/utils/format.py,sha256=yX160sGewMMZn9iJqTHRbYYt4Z9pHasjMlFT1dXrKD8,2446
@@ -192,8 +192,8 @@ verifiers/v1/utils/prime.py,sha256=UTYRjp9cbjNb6CVmBHda-1wWZAIIfxNOmTyNuT7_wL4,9
 verifiers/v1/utils/retries.py,sha256=Y2ZgrAjn-qNkRKZP_RVNL_7EK0iaRcZeCa404lpGYi4,5417
 verifiers/v1/utils/score.py,sha256=493yJVMw8teCu9JxapxMFPFzI0hNUqdo0Y2nGW4kckk,6200
 verifiers/v1/utils/version.py,sha256=-obEo_-l9-D8FLef4hYxncOe-uJpxrM1g2Hig_37Sgs,1607
-verifiers-0.3.2.dev116.dist-info/METADATA,sha256=vdkfWMbTMvOZzP2kW5qs0O09Lgxe98ObwxkMG7oZp-A,4237
-verifiers-0.3.2.dev116.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
-verifiers-0.3.2.dev116.dist-info/entry_points.txt,sha256=iugElcdWPKbQM7uFF0lZ8iUpHsNr17-BwEAAjJWxV3U,259
-verifiers-0.3.2.dev116.dist-info/licenses/LICENSE,sha256=v0RrUsdV3IDoZhrRce297IXS3xMHNJ-_LdLpFAUWb9k,1072
-verifiers-0.3.2.dev116.dist-info/RECORD,,
+verifiers-0.3.2.dev117.dist-info/METADATA,sha256=LIgiQlG8mlmytpzxUpHQpGLzD7F-bOpNG8AIKhhGyFA,4237
+verifiers-0.3.2.dev117.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
+verifiers-0.3.2.dev117.dist-info/entry_points.txt,sha256=iugElcdWPKbQM7uFF0lZ8iUpHsNr17-BwEAAjJWxV3U,259
+verifiers-0.3.2.dev117.dist-info/licenses/LICENSE,sha256=v0RrUsdV3IDoZhrRce297IXS3xMHNJ-_LdLpFAUWb9k,1072
+verifiers-0.3.2.dev117.dist-info/RECORD,,