verifiers 0.1.15.dev359__tar.gz → 0.1.15.dev364__tar.gz

{verifiers-0.1.15.dev359 → verifiers-0.1.15.dev364}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.1.15.dev359
+Version: 0.1.15.dev364
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.1.15.dev359 → verifiers-0.1.15.dev364}/verifiers/v1/cli/eval/resume.py

@@ -11,8 +11,11 @@ together), so any task that isn't fully complete is redone from scratch.
 import json
 import tomllib
 from collections import defaultdict
+from collections.abc import Iterator
 from pathlib import Path
 
+from pydantic_core import from_json
+
 from verifiers.v1.configs.eval import EvalConfig
 
 
@@ -45,54 +48,71 @@ def load_resume_config(resume_dir: Path) -> EvalConfig:
     return config
 
 
-def _read_results(results_path: Path) -> list[dict]:
-    """The previous run's traces as raw dicts. The on-disk dump carries computed fields a
-    strict `Trace` can't re-validate, so we read the JSON directly — resume only needs each
-    trace's `task.idx` and whether it `errors`ed."""
+def _read_results(results_path: Path) -> Iterator[tuple[int, int, bool]]:
+    """Stream `(file reference, task idx, errored)` without retaining decoded traces."""
     if not results_path.exists():
-        return []
-    return [
-        json.loads(line)
-        for line in results_path.read_text().split("\n")
-        if line.strip()
-    ]
+        return
+    with results_path.open("rb") as results:
+        while True:
+            offset = results.tell()
+            line = results.readline()
+            if not line:
+                break
+            if line.strip():
+                try:
+                    row = from_json(line)
+                except ValueError:
+                    row = json.loads(line)
+                yield offset, row["task"]["idx"], bool(row.get("errors"))
 
 
 def plan(
     resume_dir: Path, selected_idxs: list[int], num_rollouts: int, group: bool
-) -> tuple[list[dict], dict[int, int]]:
+) -> tuple[list[int], dict[int, int]]:
     """Diff the saved results against the run's target (`num_rollouts` per selected task).
-    Returns (rows to keep in `results.jsonl`, rollouts owed per task idx). An errored trace is
-    dropped and re-run; a group-scored task is kept only if fully complete, else its whole
-    group is redone."""
-    by_idx: dict[int, list[dict]] = defaultdict(list)
-    for row in _read_results(resume_dir / "results.jsonl"):
-        by_idx[row["task"]["idx"]].append(row)
-    keep: list[dict] = []
+    Returns (byte offsets of rows to keep, rollouts owed per task idx). An errored trace is
+    dropped and re-run; a group-scored task is kept only if fully complete, else its whole group
+    is redone."""
+    # Retain only the offsets resume can reuse; trace payloads stay on disk.
+    selected = set(selected_idxs)
+    by_idx: dict[int, list[int]] = defaultdict(list)
+    for offset, idx, errored in _read_results(resume_dir / "results.jsonl"):
+        if idx in selected and not errored and len(by_idx[idx]) < num_rollouts:
+            by_idx[idx].append(offset)
+    keep: list[int] = []
     owed: dict[int, int] = {}
     for idx in selected_idxs:
-        good = [row for row in by_idx.get(idx, []) if not row.get("errors")]
+        good = by_idx.get(idx, [])
         if group:
             if len(good) >= num_rollouts:
-                keep.extend(good[:num_rollouts])
+                keep.extend(good)
             else:
                 owed[idx] = num_rollouts  # re-run the whole group; keep none of it
         else:
-            keep.extend(good[:num_rollouts])
-            missing = num_rollouts - min(len(good), num_rollouts)
+            keep.extend(good)
+            missing = num_rollouts - len(good)
             if missing:
                 owed[idx] = missing
     return keep, owed
 
 
-def rewrite_results(resume_dir: Path, keep: list[dict]) -> None:
+def rewrite_results(resume_dir: Path, keep: list[int]) -> None:
     """Replace `results.jsonl` with just the kept (good) traces; resumed rollouts append. Via a
     temp file + atomic rename, so an interrupted resume can't corrupt the prior good results."""
     path = resume_dir / "results.jsonl"
     tmp = path.with_suffix(".jsonl.tmp")
-    with tmp.open("w") as f:
-        for row in keep:
-            f.write(json.dumps(row) + "\n")
+    if not keep:
+        tmp.write_bytes(b"")
+        tmp.replace(path)
+        return
+    # Re-read retained rows by offset while building the atomic replacement.
+    with path.open("rb") as results, tmp.open("wb") as output:
+        for offset in keep:
+            results.seek(offset)
+            raw = results.readline()
+            output.write(raw)
+            if not raw.endswith(b"\n"):
+                output.write(b"\n")
     tmp.replace(path)
 
 

{verifiers-0.1.15.dev359 → verifiers-0.1.15.dev364}/verifiers/v1/interception/server.py

@@ -24,12 +24,12 @@ import contextlib
 import json
 import logging
 import secrets
-from collections.abc import Awaitable, Callable
+from collections.abc import AsyncIterator, Awaitable, Callable
 from dataclasses import dataclass, field
 from typing import TYPE_CHECKING
 
 from aiohttp import web
-from pydantic import ValidationError
+from pydantic import TypeAdapter, ValidationError
 from pydantic_core import PydanticSerializationError, from_json, to_json
 
 from verifiers.v1.clients import RolloutContext
@@ -56,6 +56,7 @@ logger = logging.getLogger(__name__)
 # context window are the real limits, this is just a host-OOM backstop.
 _MAX_REQUEST_BODY = 1024**3  # 1 GiB (aiohttp's default is 1 MiB)
 _KEEPALIVE_INTERVAL_SECONDS = 3
+_STREAM_QUEUE_MAXSIZE = 16
 # The server binds loopback; callers reach it via localhost or a host tunnel (see `reachable_url`).
 _HOST = "127.0.0.1"
 
@@ -69,6 +70,20 @@ def _completion_response(completion: dict | None) -> web.Response:
     return web.Response(body=body, content_type="application/json", charset="utf-8")
 
 
+async def _queue_chunks(
+    chunks: AsyncIterator[bytes],
+    queue: asyncio.Queue[bytes | None],
+    ready: asyncio.Event,
+) -> None:
+    try:
+        async for chunk in chunks:
+            await queue.put(chunk)
+            ready.set()
+    finally:
+        await queue.put(None)
+        ready.set()
+
+
 @dataclass(frozen=True)
 class RolloutLimits:
     """Per-rollout framework limits (None = no cap), checked before each turn is served.
@@ -446,28 +461,37 @@ class InterceptionServer:
         )
         resp.content_type = reply.content_type.split(";")[0].strip()
         buffer = bytearray()
-        chunks = reply.chunks.__aiter__()
-        next_chunk = asyncio.create_task(anext(chunks, None))
+        # One bounded producer avoids per-event tasks; keepalive timeouts only cancel readiness waits.
+        queue: asyncio.Queue[bytes | None] = asyncio.Queue(
+            maxsize=_STREAM_QUEUE_MAXSIZE
+        )
+        ready = asyncio.Event()
+        producer = asyncio.create_task(_queue_chunks(reply.chunks, queue, ready))
         try:
             await resp.prepare(request)
             while True:
-                done, _ = await asyncio.wait(
-                    {next_chunk}, timeout=_KEEPALIVE_INTERVAL_SECONDS
-                )
-                if not done:
+                try:
+                    async with asyncio.timeout(_KEEPALIVE_INTERVAL_SECONDS):
+                        await ready.wait()
+                except TimeoutError:
                     await resp.write(b": keepalive\n\n")
                     continue
-                chunk = next_chunk.result()
+                chunk = queue.get_nowait()
+                if queue.empty():
+                    ready.clear()
                 if chunk is None:
+                    await producer
                     break
                 buffer += chunk
                 await resp.write(chunk)
-                next_chunk = asyncio.create_task(anext(chunks, None))
         except ConnectionResetError:
             return resp
         finally:
-            next_chunk.cancel()
-            await asyncio.gather(next_chunk, return_exceptions=True)
+            producer.cancel()
+            # Let a canceled producer enqueue EOF while unwinding.
+            if queue.full():
+                queue.get_nowait()
+            await asyncio.gather(producer, return_exceptions=True)
             await reply.close()
 
         try:
@@ -522,8 +546,12 @@ class InterceptionServer:
         if session is None:
             return web.json_response({"error": "unauthorized"}, status=401)
         logger.debug("intercept GET /state: id=%s", session.trace.id)
+        state = session.trace.state
         return web.Response(
-            text=session.trace.state.model_dump_json(), content_type="application/json"
+            # TypeAdapter emits UTF-8 bytes directly, avoiding a JSON str copy in aiohttp.
+            body=TypeAdapter(type(state)).dump_json(state),
+            content_type="application/json",
+            charset="utf-8",
         )
 
     async def handle_task_get(self, request: web.Request) -> web.Response:
@@ -550,13 +578,11 @@ class InterceptionServer:
             return web.json_response({"error": "unauthorized"}, status=401)
         logger.debug("intercept PUT /state: id=%s", session.trace.id)
         state_cls = type(session.trace.state)
+        raw = await request.read()
         try:
-            new_state = state_cls.model_validate(await request.json())
-        except (ValidationError, ValueError) as e:
-            # Malformed JSON (`request.json()` -> JSONDecodeError, a ValueError) or a pushed state
-            # that doesn't fit the trace's `State` type (almost always a `StateT` mismatch between the
-            # taskset and a server). Surface a clean 400 (with the reason) rather than a 500, so the
-            # server's failed PUT fails the rollout legibly.
+            new_state = state_cls.model_validate_json(raw)
+        except ValidationError as e:
+            # Reject malformed, over-nested, or mismatched state before it enters the shared channel.
             logger.warning("state PUT rejected: id=%s %s", session.trace.id, e)
             return web.json_response(
                 {"error": f"invalid state PUT for {state_cls.__name__}: {e}"},

{verifiers-0.1.15.dev359 → verifiers-0.1.15.dev364}/verifiers/v1/mcp/server.py

@@ -12,18 +12,21 @@ import contextlib
 import contextvars
 import functools
 import inspect
+import logging
 import os
 from typing import TYPE_CHECKING, Callable, ClassVar, Generic, TypeVar, get_args
 
+from pydantic import TypeAdapter, ValidationError
 from pydantic_config import BaseConfig
 
 from verifiers.v1.state import State, StateT, state_cls
 
 if TYPE_CHECKING:
-    from httpx import AsyncClient
+    from httpx import AsyncClient, Response
     from mcp.server.fastmcp import FastMCP
 
 ConfigT = TypeVar("ConfigT", bound=BaseConfig)
+logger = logging.getLogger(__name__)
 
 STATE_TIMEOUT = 30.0
 """Seconds for a state-channel GET/PUT (localhost, or a tunnel to the host)."""
@@ -38,13 +41,13 @@ async def _channel_request(
     url: str,
     secret: str,
     *,
-    json: dict | None = None,
+    content: bytes | None = None,
     client: AsyncClient | None = None,
-) -> dict:
+) -> Response:
     """One bearer-authed call to the interception `/state` or `/task` channel, retried with
     exponential backoff + jitter on transient failures (connection resets / 5xx from the host
     tunnel). A 4xx is a real error (e.g. an invalid state PUT) and is not retried. Returns the
-    parsed JSON body."""
+    response so typed callers can validate its JSON bytes directly."""
     import httpx
     from tenacity import (
         AsyncRetrying,
@@ -58,18 +61,21 @@ async def _channel_request(
             isinstance(e, httpx.HTTPStatusError) and e.response.status_code >= 500
         )
 
-    async def request() -> dict:
+    async def request() -> Response:
         manager = (
             contextlib.nullcontext(client)
             if client is not None
             else httpx.AsyncClient(timeout=STATE_TIMEOUT)
         )
         async with manager as request_client:
+            headers = {"Authorization": f"Bearer {secret}"}
+            if content is not None:
+                headers["Content-Type"] = "application/json"
             resp = await request_client.request(
-                method, url, json=json, headers={"Authorization": f"Bearer {secret}"}
+                method, url, content=content, headers=headers
             )
             resp.raise_for_status()
-            return resp.json()
+            return resp
 
     return await AsyncRetrying(
         stop=stop_after_attempt(STATE_RETRIES + 1),
@@ -158,6 +164,8 @@ class ServerBase(Generic[ConfigT, StateT]):
     def __init__(self, config: ConfigT) -> None:
         self.config = config
         self._state_cls = state_cls(type(self))
+        # Reuse Pydantic's public byte serializer/validator across state-channel calls.
+        self._state_adapter = TypeAdapter(self._state_cls)
         self._inert_state: StateT = self._state_cls()  # type: ignore[assignment]
         """A fresh state used outside a tool/respond call (setup, or a manual debug run) — there's
         no channel to sync, so writes to it don't escape the process (matches the no-channel case)."""
@@ -188,27 +196,31 @@ class ServerBase(Generic[ConfigT, StateT]):
         """Fetch the in-flight call's shared state from the channel (so it reflects other servers'
         writes); a fresh inert state when there's no channel."""
         url, secret = self._state_channel()
-        cls = self._state_cls
         if not url:
-            return cls()
-        return cls.model_validate(
-            await _channel_request("GET", url, secret, client=self._state_client)
-        )
+            return self._state_cls()
+        response = await _channel_request("GET", url, secret, client=self._state_client)
+        try:
+            return self._state_adapter.validate_json(response.content)
+        except ValidationError as e:
+            # Excessively nested or otherwise invalid state is a broken channel contract.
+            logger.warning(
+                "state pull rejected for %s: %s", self._state_cls.__name__, e
+            )
+            raise
 
-    async def _push_state(self, before: dict) -> None:
+    async def _push_state(self, before: bytes) -> None:
         """Push the in-flight call's `self.state` back to the shared channel if it changed. No-op
         without a channel or when nothing changed."""
         url, secret = self._state_channel()
         if not url:
             return
         state = _call_state.get()
-        after = (state if state is not None else self._inert_state).model_dump(
-            mode="json"
-        )
+        current = state if state is not None else self._inert_state
+        after = self._state_adapter.dump_json(current)
         if after == before:
             return
         await _channel_request(
-            "PUT", url, secret, json=after, client=self._state_client
+            "PUT", url, secret, content=after, client=self._state_client
         )
 
     async def _fetch_task(self, state_url: str | None, secret: str):
@@ -222,7 +234,7 @@ class ServerBase(Generic[ConfigT, StateT]):
             if state_url.endswith("/state")
             else state_url
         )
-        data = await _channel_request("GET", task_url, secret)
+        data = (await _channel_request("GET", task_url, secret)).json()
         return _import_ref(data["cls"]).model_validate_json(data["task"])
 
     async def _setup_task_from_channel(
@@ -253,7 +265,8 @@ class ServerBase(Generic[ConfigT, StateT]):
             state = await self._pull_state()
             token = _call_state.set(state)
             try:
-                before = state.model_dump(mode="json")
+                # Reuse the second serialization as the PUT body when the callback mutates state.
+                before = self._state_adapter.dump_json(state)
                 result = fn(*args, **kwargs)
                 if inspect.isawaitable(result):
                     result = await result

{verifiers-0.1.15.dev359 → verifiers-0.1.15.dev364}/verifiers/v1/runtimes/base.py

@@ -322,8 +322,18 @@ async def host_endpoint(port: int, is_local: bool, labels: list[str] | None = No
     try:
         yield url
     finally:
+        # Delay cancellation until the synchronous stop has finished.
+        cancelled = None
+        stop_task = asyncio.create_task(asyncio.to_thread(tunnel.sync_stop))
         with contextlib.suppress(Exception):
-            tunnel.sync_stop()
+            while not stop_task.done():
+                try:
+                    await asyncio.shield(stop_task)
+                except asyncio.CancelledError as e:
+                    cancelled = e
+            stop_task.result()
+        if cancelled is not None:
+            raise cancelled
 
 
 class _Host:

{verifiers-0.1.15.dev359 → verifiers-0.1.15.dev364}/verifiers/v1/runtimes/prime.py

@@ -7,11 +7,11 @@ direction (a program in the sandbox reaching a host service) is the shared host-
 """
 
 import asyncio
-import base64
 import contextlib
 import logging
 import shlex
-from pathlib import PurePosixPath
+import tempfile
+from pathlib import Path, PurePosixPath
 from typing import ClassVar, Literal
 
 from pydantic_config import BaseConfig
@@ -176,10 +176,22 @@ class PrimeRuntime(Runtime):
             )
 
     async def read(self, path: str) -> bytes:
-        result = await self.run(["sh", "-c", f"base64 {shlex.quote(path)}"], {})
-        if result.exit_code != 0:
-            raise SandboxError(f"read {path!r}: {result.stderr.strip()}")
-        return base64.b64decode(result.stdout)
+        # Avoid background-job log limits and base64 overhead by downloading binary data directly.
+        # The temporary file is removed on every exit, and its byte read stays off the event loop.
+        target = (
+            path
+            if path.startswith("/")
+            else f"{self.config.workdir.rstrip('/')}/{path}"
+        )
+        try:
+            with tempfile.TemporaryDirectory() as directory:
+                download = Path(directory) / "download"
+                await self._client.download_file(
+                    self._sandbox_id, target, str(download)
+                )
+                return await asyncio.to_thread(download.read_bytes)
+        except Exception as e:
+            raise SandboxError(f"read {path!r}: {e}") from e
 
     async def write(self, path: str, data: bytes) -> None:
         # Upload via the gateway (multipart) — never inline the bytes on the command line