verifiers 0.1.15.dev360__tar.gz → 0.1.15.dev364__tar.gz

{verifiers-0.1.15.dev360 → verifiers-0.1.15.dev364}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.1.15.dev360
+Version: 0.1.15.dev364
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.1.15.dev360 → verifiers-0.1.15.dev364}/verifiers/v1/cli/eval/resume.py

@@ -11,8 +11,11 @@ together), so any task that isn't fully complete is redone from scratch.
 import json
 import tomllib
 from collections import defaultdict
+from collections.abc import Iterator
 from pathlib import Path
 
+from pydantic_core import from_json
+
 from verifiers.v1.configs.eval import EvalConfig
 
 
@@ -45,54 +48,71 @@ def load_resume_config(resume_dir: Path) -> EvalConfig:
     return config
 
 
-def _read_results(results_path: Path) -> list[dict]:
-    """The previous run's traces as raw dicts. The on-disk dump carries computed fields a
-    strict `Trace` can't re-validate, so we read the JSON directly — resume only needs each
-    trace's `task.idx` and whether it `errors`ed."""
+def _read_results(results_path: Path) -> Iterator[tuple[int, int, bool]]:
+    """Stream `(file reference, task idx, errored)` without retaining decoded traces."""
     if not results_path.exists():
-        return []
-    return [
-        json.loads(line)
-        for line in results_path.read_text().split("\n")
-        if line.strip()
-    ]
+        return
+    with results_path.open("rb") as results:
+        while True:
+            offset = results.tell()
+            line = results.readline()
+            if not line:
+                break
+            if line.strip():
+                try:
+                    row = from_json(line)
+                except ValueError:
+                    row = json.loads(line)
+                yield offset, row["task"]["idx"], bool(row.get("errors"))
 
 
 def plan(
     resume_dir: Path, selected_idxs: list[int], num_rollouts: int, group: bool
-) -> tuple[list[dict], dict[int, int]]:
+) -> tuple[list[int], dict[int, int]]:
     """Diff the saved results against the run's target (`num_rollouts` per selected task).
-    Returns (rows to keep in `results.jsonl`, rollouts owed per task idx). An errored trace is
-    dropped and re-run; a group-scored task is kept only if fully complete, else its whole
-    group is redone."""
-    by_idx: dict[int, list[dict]] = defaultdict(list)
-    for row in _read_results(resume_dir / "results.jsonl"):
-        by_idx[row["task"]["idx"]].append(row)
-    keep: list[dict] = []
+    Returns (byte offsets of rows to keep, rollouts owed per task idx). An errored trace is
+    dropped and re-run; a group-scored task is kept only if fully complete, else its whole group
+    is redone."""
+    # Retain only the offsets resume can reuse; trace payloads stay on disk.
+    selected = set(selected_idxs)
+    by_idx: dict[int, list[int]] = defaultdict(list)
+    for offset, idx, errored in _read_results(resume_dir / "results.jsonl"):
+        if idx in selected and not errored and len(by_idx[idx]) < num_rollouts:
+            by_idx[idx].append(offset)
+    keep: list[int] = []
     owed: dict[int, int] = {}
     for idx in selected_idxs:
-        good = [row for row in by_idx.get(idx, []) if not row.get("errors")]
+        good = by_idx.get(idx, [])
         if group:
             if len(good) >= num_rollouts:
-                keep.extend(good[:num_rollouts])
+                keep.extend(good)
             else:
                 owed[idx] = num_rollouts  # re-run the whole group; keep none of it
         else:
-            keep.extend(good[:num_rollouts])
-            missing = num_rollouts - min(len(good), num_rollouts)
+            keep.extend(good)
+            missing = num_rollouts - len(good)
             if missing:
                 owed[idx] = missing
     return keep, owed
 
 
-def rewrite_results(resume_dir: Path, keep: list[dict]) -> None:
+def rewrite_results(resume_dir: Path, keep: list[int]) -> None:
     """Replace `results.jsonl` with just the kept (good) traces; resumed rollouts append. Via a
     temp file + atomic rename, so an interrupted resume can't corrupt the prior good results."""
     path = resume_dir / "results.jsonl"
     tmp = path.with_suffix(".jsonl.tmp")
-    with tmp.open("w") as f:
-        for row in keep:
-            f.write(json.dumps(row) + "\n")
+    if not keep:
+        tmp.write_bytes(b"")
+        tmp.replace(path)
+        return
+    # Re-read retained rows by offset while building the atomic replacement.
+    with path.open("rb") as results, tmp.open("wb") as output:
+        for offset in keep:
+            results.seek(offset)
+            raw = results.readline()
+            output.write(raw)
+            if not raw.endswith(b"\n"):
+                output.write(b"\n")
     tmp.replace(path)
 
 

{verifiers-0.1.15.dev360 → verifiers-0.1.15.dev364}/verifiers/v1/interception/server.py

@@ -24,7 +24,7 @@ import contextlib
 import json
 import logging
 import secrets
-from collections.abc import Awaitable, Callable
+from collections.abc import AsyncIterator, Awaitable, Callable
 from dataclasses import dataclass, field
 from typing import TYPE_CHECKING
 
@@ -56,6 +56,7 @@ logger = logging.getLogger(__name__)
 # context window are the real limits, this is just a host-OOM backstop.
 _MAX_REQUEST_BODY = 1024**3  # 1 GiB (aiohttp's default is 1 MiB)
 _KEEPALIVE_INTERVAL_SECONDS = 3
+_STREAM_QUEUE_MAXSIZE = 16
 # The server binds loopback; callers reach it via localhost or a host tunnel (see `reachable_url`).
 _HOST = "127.0.0.1"
 
@@ -69,6 +70,20 @@ def _completion_response(completion: dict | None) -> web.Response:
     return web.Response(body=body, content_type="application/json", charset="utf-8")
 
 
+async def _queue_chunks(
+    chunks: AsyncIterator[bytes],
+    queue: asyncio.Queue[bytes | None],
+    ready: asyncio.Event,
+) -> None:
+    try:
+        async for chunk in chunks:
+            await queue.put(chunk)
+            ready.set()
+    finally:
+        await queue.put(None)
+        ready.set()
+
+
 @dataclass(frozen=True)
 class RolloutLimits:
     """Per-rollout framework limits (None = no cap), checked before each turn is served.
@@ -446,28 +461,37 @@ class InterceptionServer:
         )
         resp.content_type = reply.content_type.split(";")[0].strip()
         buffer = bytearray()
-        chunks = reply.chunks.__aiter__()
-        next_chunk = asyncio.create_task(anext(chunks, None))
+        # One bounded producer avoids per-event tasks; keepalive timeouts only cancel readiness waits.
+        queue: asyncio.Queue[bytes | None] = asyncio.Queue(
+            maxsize=_STREAM_QUEUE_MAXSIZE
+        )
+        ready = asyncio.Event()
+        producer = asyncio.create_task(_queue_chunks(reply.chunks, queue, ready))
         try:
             await resp.prepare(request)
             while True:
-                done, _ = await asyncio.wait(
-                    {next_chunk}, timeout=_KEEPALIVE_INTERVAL_SECONDS
-                )
-                if not done:
+                try:
+                    async with asyncio.timeout(_KEEPALIVE_INTERVAL_SECONDS):
+                        await ready.wait()
+                except TimeoutError:
                     await resp.write(b": keepalive\n\n")
                     continue
-                chunk = next_chunk.result()
+                chunk = queue.get_nowait()
+                if queue.empty():
+                    ready.clear()
                 if chunk is None:
+                    await producer
                     break
                 buffer += chunk
                 await resp.write(chunk)
-                next_chunk = asyncio.create_task(anext(chunks, None))
         except ConnectionResetError:
             return resp
         finally:
-            next_chunk.cancel()
-            await asyncio.gather(next_chunk, return_exceptions=True)
+            producer.cancel()
+            # Let a canceled producer enqueue EOF while unwinding.
+            if queue.full():
+                queue.get_nowait()
+            await asyncio.gather(producer, return_exceptions=True)
             await reply.close()
 
         try:

{verifiers-0.1.15.dev360 → verifiers-0.1.15.dev364}/verifiers/v1/runtimes/base.py

@@ -322,8 +322,18 @@ async def host_endpoint(port: int, is_local: bool, labels: list[str] | None = No
     try:
         yield url
     finally:
+        # Delay cancellation until the synchronous stop has finished.
+        cancelled = None
+        stop_task = asyncio.create_task(asyncio.to_thread(tunnel.sync_stop))
         with contextlib.suppress(Exception):
-            tunnel.sync_stop()
+            while not stop_task.done():
+                try:
+                    await asyncio.shield(stop_task)
+                except asyncio.CancelledError as e:
+                    cancelled = e
+            stop_task.result()
+        if cancelled is not None:
+            raise cancelled
 
 
 class _Host:

{verifiers-0.1.15.dev360 → verifiers-0.1.15.dev364}/verifiers/v1/runtimes/prime.py

@@ -7,11 +7,11 @@ direction (a program in the sandbox reaching a host service) is the shared host-
 """
 
 import asyncio
-import base64
 import contextlib
 import logging
 import shlex
-from pathlib import PurePosixPath
+import tempfile
+from pathlib import Path, PurePosixPath
 from typing import ClassVar, Literal
 
 from pydantic_config import BaseConfig
@@ -176,10 +176,22 @@ class PrimeRuntime(Runtime):
             )
 
     async def read(self, path: str) -> bytes:
-        result = await self.run(["sh", "-c", f"base64 {shlex.quote(path)}"], {})
-        if result.exit_code != 0:
-            raise SandboxError(f"read {path!r}: {result.stderr.strip()}")
-        return base64.b64decode(result.stdout)
+        # Avoid background-job log limits and base64 overhead by downloading binary data directly.
+        # The temporary file is removed on every exit, and its byte read stays off the event loop.
+        target = (
+            path
+            if path.startswith("/")
+            else f"{self.config.workdir.rstrip('/')}/{path}"
+        )
+        try:
+            with tempfile.TemporaryDirectory() as directory:
+                download = Path(directory) / "download"
+                await self._client.download_file(
+                    self._sandbox_id, target, str(download)
+                )
+                return await asyncio.to_thread(download.read_bytes)
+        except Exception as e:
+            raise SandboxError(f"read {path!r}: {e}") from e
 
     async def write(self, path: str, data: bytes) -> None:
         # Upload via the gateway (multipart) — never inline the bytes on the command line