verifiers 0.3.2.dev116__tar.gz → 0.3.2.dev118__tar.gz

{verifiers-0.3.2.dev116 → verifiers-0.3.2.dev118}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.3.2.dev116
+Version: 0.3.2.dev118
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.3.2.dev116 → verifiers-0.3.2.dev118}/tests/v1/test_trace.py

@@ -10,6 +10,7 @@ import pytest
 
 import verifiers.v1 as vf
 from verifiers.v1.agent import Interaction
+from verifiers.v1.cli.output import write_episode
 from verifiers.v1.graph import MessageNode
 from verifiers.v1.harnesses.rlm.harness import (
     RLM_SESSION_METADATA_KEY,
@@ -144,13 +145,13 @@ def test_bare_trace_round_trip():
     assert rt.reward == 0.0 and rt.errors == []
 
 
-def test_custom_task_state_round_trip():
+def test_custom_task_state_round_trip(tmp_path):
     # Custom data and state round-trip into the same parameterization. Data fields are
     # typed (not just `model_extra`); `state` is runtime-only and never crosses the wire.
     tr = vf.Trace[MyTask, MyState](
         agent=vf.AgentInfo(config=vf.AgentConfig()),
         task=vf.TraceTask(type="MyTask", data=MyTask(idx=0, prompt="q", answer="gold")),
-        state=MyState(score=7),
+        state=MyState(score=7, artifacts={"/artifact.bin": b"\xff\x00\xfe"}),
         nodes=[
             MessageNode(parent=None, message=UserMessage(content="q"), sampled=False),
             MessageNode(parent=0, message=AssistantMessage(content="a"), sampled=True),
@@ -167,6 +168,10 @@ def test_custom_task_state_round_trip():
     assert rt.task.type == "MyTask"  # the producing class's name survives the wire
     assert rt.num_turns == 1 and rt.num_branches == 1
     assert rt.reward == 0.5  # property recomputed from `rewards`
+    write_episode(tmp_path, vf.Episode(task=tr.task, traces=[tr], ok=True))
+    saved = json.loads((tmp_path / "traces.jsonl").read_bytes())
+    assert "state" not in saved["traces"][0]
+    assert tr.state.artifacts["/artifact.bin"] == b"\xff\x00\xfe"
 
 
 def test_wire_trace_round_trip():

verifiers-0.3.2.dev118/verifiers/__init__.py

@@ -0,0 +1,9 @@
+"""The verifiers package root; the stack lives in `verifiers.v1` (`import verifiers.v1 as vf`)."""
+
+from importlib.metadata import PackageNotFoundError
+from importlib.metadata import version as _version
+
+try:
+    __version__ = _version("verifiers")
+except PackageNotFoundError:  # source tree without install metadata
+    __version__ = "0.0.0+unknown"

{verifiers-0.3.2.dev116 → verifiers-0.3.2.dev118}/verifiers/v1/rollout.py

@@ -485,7 +485,9 @@ class Rollout:
                         )
                         if self._collect_artifacts and not trace.state.artifacts:
                             trace.state.artifacts = await collect(
-                                runtime, self.task.data.artifacts
+                                runtime,
+                                self.task.data.artifacts,
+                                max_bytes=self.task.data.artifact_max_bytes,
                             )
                 now = time.time()
                 trace.timing.finalize.end = now

{verifiers-0.3.2.dev116 → verifiers-0.3.2.dev118}/verifiers/v1/runtimes/prime.py

@@ -1,8 +1,8 @@
 """Remote Prime VM sandbox runtime."""
 
 import asyncio
-import base64
 import contextlib
+import io
 import logging
 import math
 import shlex
@@ -329,19 +329,30 @@ class PrimeRuntime(Runtime):
     async def _read(self, path: str, max_bytes: int | None = None) -> bytes:
         if max_bytes is not None:
             try:
-                # VM execute_command uses bash and returns the complete output stream.
-                result = await self._client.execute_command(
+                # Stream binary output: execute_command buffers base64 text for the
+                # entire file. Bound the source read and the host buffer independently.
+                process = await self._client.open_process(
                     self.info.id,
-                    f"set -o pipefail; head -c {max_bytes} -- {shlex.quote(path)} | base64",
+                    f"head -c {max_bytes} -- {shlex.quote(path)}",
                     working_dir=self.config.workdir,
                     env=self.process_env({}),
-                    timeout=EFFECTIVELY_UNBOUNDED_SECONDS,
                 )
+                async with contextlib.aclosing(process):
+                    with io.BytesIO() as data:
+                        async for chunk in process.stdout:
+                            if data.tell() + len(chunk) > max_bytes:
+                                raise SandboxError(
+                                    "read stream exceeded its byte limit"
+                                )
+                            data.write(chunk)
+                        stderr = b""
+                        async for chunk in process.stderr:
+                            stderr = (stderr + chunk)[-500:]
+                        if await process.wait():
+                            raise SandboxError(stderr.decode(errors="replace").strip())
+                        return data.getvalue()
             except Exception as exc:
                 raise SandboxError(f"read {path!r}: {exc}") from exc
-            if result.exit_code:
-                raise SandboxError(f"read {path!r}: {result.stderr.strip()[-500:]}")
-            return base64.b64decode(result.stdout)
         # Avoid background-job log limits and base64 overhead by downloading binary data directly.
         # The temporary file is removed on every exit, and its byte read stays off the event loop.
         target = (

{verifiers-0.3.2.dev116 → verifiers-0.3.2.dev118}/verifiers/v1/task.py

@@ -25,7 +25,7 @@ from verifiers.v1.configs.task import TaskConfig
 from verifiers.v1.errors import TaskError, boundary
 from verifiers.v1.state import StateT
 from verifiers.v1.types import Messages, content_text
-from verifiers.v1.utils.artifacts import Artifact
+from verifiers.v1.utils.artifacts import MAX_ARTIFACT_BYTES, Artifact
 from verifiers.v1.utils.decorators import (
     discover_decorated,
     invoke_all,
@@ -115,6 +115,8 @@ class TaskData(BaseModel):
     on top of the implicitly collected `/logs/artifacts/` convention dir. Declare
     runtime outputs that must cross that boundary. A declared path that is missing at
     collection time fails the rollout."""
+    artifact_max_bytes: int = Field(MAX_ARTIFACT_BYTES, gt=0)
+    """Total byte limit for collected artifact archives, including the convention dir."""
 
     timeout: TaskTimeout = TaskTimeout()
     resources: TaskResources = TaskResources()

{verifiers-0.3.2.dev116 → verifiers-0.3.2.dev118}/verifiers/v1/tasksets/harbor/taskset.py

@@ -40,7 +40,7 @@ from verifiers.v1.state import State
 from verifiers.v1.task import Task, TaskData, TaskResources, TaskTimeout
 from verifiers.v1.taskset import Taskset
 from verifiers.v1.trace import Trace
-from verifiers.v1.utils.artifacts import Artifact, collect
+from verifiers.v1.utils.artifacts import MAX_ARTIFACT_BYTES, Artifact, collect
 from verifiers.v1.utils.decorators import reward
 
 logger = logging.getLogger(__name__)
@@ -61,6 +61,8 @@ class HarborTaskConfig(TaskConfig):
 
 
 class HarborConfig(TasksetConfig):
+    artifact_max_bytes: int = Field(MAX_ARTIFACT_BYTES, gt=0)
+    """Total byte limit for artifact archives transferred out of each solver runtime."""
     task: HarborTaskConfig = HarborTaskConfig()
     dataset: str = "harbor/hello-world"
     """A Harbor Hub package id ("org/name" or "org/name@ref"), where ref is a
@@ -272,7 +274,9 @@ class HarborTask(Task[HarborData, State, HarborTaskConfig]):
                     f"{hook.command}\n{detail}"
                 )
         if not self.scoring_deferred:
-            trace.state.artifacts = await collect(runtime, self.data.artifacts)
+            trace.state.artifacts = await collect(
+                runtime, self.data.artifacts, max_bytes=self.data.artifact_max_bytes
+            )
 
     async def stage_verifier(self, trace: Trace, runtime: Runtime) -> None:
         if any(
@@ -582,6 +586,7 @@ def parse_task(task_dir: Path, idx: int, harbor_config: HarborConfig) -> HarborD
         )
     return HarborData(
         idx=idx,
+        artifact_max_bytes=harbor_config.artifact_max_bytes,
         name=harbor_task.name,
         description=task.description if task else None,
         prompt=harbor_task.instruction.strip(),

{verifiers-0.3.2.dev116 → verifiers-0.3.2.dev118}/verifiers/v1/utils/artifacts.py

@@ -21,7 +21,7 @@ ARTIFACTS_DIR = "/logs/artifacts"
 """Implicit artifact directory; tasks that write here need no declaration."""
 
 MAX_ARTIFACT_BYTES = 32 * 1024 * 1024
-"""Ceiling per collection. Sized for a delta, not a tree: the grading box boots from the
+"""Default ceiling per collection. Sized for a delta, not a tree: the grading box boots from the
 agent's image, so the repo is already there and only its output has to travel."""
 
 
@@ -35,7 +35,10 @@ class Artifact(BaseModel):
 
 
 async def collect(
-    runtime: Runtime, artifacts: list[Artifact] | None = None
+    runtime: Runtime,
+    artifacts: list[Artifact] | None = None,
+    *,
+    max_bytes: int = MAX_ARTIFACT_BYTES,
 ) -> dict[str, bytes | None]:
     """Tar the convention dir and every declared path out of `runtime`.
 
@@ -114,7 +117,7 @@ async def collect(
         )
         existence.extend(output.splitlines())
     collected: dict[str, bytes | None] = {}
-    budget = MAX_ARTIFACT_BYTES
+    budget = max_bytes
     for artifact, exists in zip(entries, existence, strict=True):
         source = artifact.source
         if exists != "1":

verifiers-0.3.2.dev116/verifiers/__init__.py

@@ -1,14 +0,0 @@
-"""The verifiers package root.
-
-The v1 stack lives in `verifiers.v1` (`import verifiers.v1 as vf`). The
-classic v0 stack (`verifiers.legacy`, which also answered at its historical
-top-level paths — `verifiers.envs`, `verifiers.types`, ...) has been removed.
-"""
-
-from importlib.metadata import PackageNotFoundError
-from importlib.metadata import version as _version
-
-try:
-    __version__ = _version("verifiers")
-except PackageNotFoundError:  # source tree without install metadata
-    __version__ = "0.0.0+unknown"

verifiers-0.3.2.dev116/verifiers/cli/commands/init.py

@@ -1,6 +0,0 @@
-"""Environment initialization command module for external hosts."""
-
-from verifiers.v1.cli.init import main
-
-if __name__ == "__main__":
-    main()