verifiers 0.1.15.dev360__tar.gz → 0.1.15.dev366__tar.gz

{verifiers-0.1.15.dev360 → verifiers-0.1.15.dev366}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.1.15.dev360
+Version: 0.1.15.dev366
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.1.15.dev360 → verifiers-0.1.15.dev366}/verifiers/v1/cli/eval/resume.py

@@ -11,8 +11,11 @@ together), so any task that isn't fully complete is redone from scratch.
 import json
 import tomllib
 from collections import defaultdict
+from collections.abc import Iterator
 from pathlib import Path
 
+from pydantic_core import from_json
+
 from verifiers.v1.configs.eval import EvalConfig
 
 
@@ -45,54 +48,71 @@ def load_resume_config(resume_dir: Path) -> EvalConfig:
     return config
 
 
-def _read_results(results_path: Path) -> list[dict]:
-    """The previous run's traces as raw dicts. The on-disk dump carries computed fields a
-    strict `Trace` can't re-validate, so we read the JSON directly — resume only needs each
-    trace's `task.idx` and whether it `errors`ed."""
+def _read_results(results_path: Path) -> Iterator[tuple[int, int, bool]]:
+    """Stream `(file reference, task idx, errored)` without retaining decoded traces."""
     if not results_path.exists():
-        return []
-    return [
-        json.loads(line)
-        for line in results_path.read_text().split("\n")
-        if line.strip()
-    ]
+        return
+    with results_path.open("rb") as results:
+        while True:
+            offset = results.tell()
+            line = results.readline()
+            if not line:
+                break
+            if line.strip():
+                try:
+                    row = from_json(line)
+                except ValueError:
+                    row = json.loads(line)
+                yield offset, row["task"]["idx"], bool(row.get("errors"))
 
 
 def plan(
     resume_dir: Path, selected_idxs: list[int], num_rollouts: int, group: bool
-) -> tuple[list[dict], dict[int, int]]:
+) -> tuple[list[int], dict[int, int]]:
     """Diff the saved results against the run's target (`num_rollouts` per selected task).
-    Returns (rows to keep in `results.jsonl`, rollouts owed per task idx). An errored trace is
-    dropped and re-run; a group-scored task is kept only if fully complete, else its whole
-    group is redone."""
-    by_idx: dict[int, list[dict]] = defaultdict(list)
-    for row in _read_results(resume_dir / "results.jsonl"):
-        by_idx[row["task"]["idx"]].append(row)
-    keep: list[dict] = []
+    Returns (byte offsets of rows to keep, rollouts owed per task idx). An errored trace is
+    dropped and re-run; a group-scored task is kept only if fully complete, else its whole group
+    is redone."""
+    # Retain only the offsets resume can reuse; trace payloads stay on disk.
+    selected = set(selected_idxs)
+    by_idx: dict[int, list[int]] = defaultdict(list)
+    for offset, idx, errored in _read_results(resume_dir / "results.jsonl"):
+        if idx in selected and not errored and len(by_idx[idx]) < num_rollouts:
+            by_idx[idx].append(offset)
+    keep: list[int] = []
     owed: dict[int, int] = {}
     for idx in selected_idxs:
-        good = [row for row in by_idx.get(idx, []) if not row.get("errors")]
+        good = by_idx.get(idx, [])
         if group:
             if len(good) >= num_rollouts:
-                keep.extend(good[:num_rollouts])
+                keep.extend(good)
             else:
                 owed[idx] = num_rollouts  # re-run the whole group; keep none of it
         else:
-            keep.extend(good[:num_rollouts])
-            missing = num_rollouts - min(len(good), num_rollouts)
+            keep.extend(good)
+            missing = num_rollouts - len(good)
             if missing:
                 owed[idx] = missing
     return keep, owed
 
 
-def rewrite_results(resume_dir: Path, keep: list[dict]) -> None:
+def rewrite_results(resume_dir: Path, keep: list[int]) -> None:
     """Replace `results.jsonl` with just the kept (good) traces; resumed rollouts append. Via a
     temp file + atomic rename, so an interrupted resume can't corrupt the prior good results."""
     path = resume_dir / "results.jsonl"
     tmp = path.with_suffix(".jsonl.tmp")
-    with tmp.open("w") as f:
-        for row in keep:
-            f.write(json.dumps(row) + "\n")
+    if not keep:
+        tmp.write_bytes(b"")
+        tmp.replace(path)
+        return
+    # Re-read retained rows by offset while building the atomic replacement.
+    with path.open("rb") as results, tmp.open("wb") as output:
+        for offset in keep:
+            results.seek(offset)
+            raw = results.readline()
+            output.write(raw)
+            if not raw.endswith(b"\n"):
+                output.write(b"\n")
     tmp.replace(path)
 
 

{verifiers-0.1.15.dev360 → verifiers-0.1.15.dev366}/verifiers/v1/dialects/anthropic.py

@@ -122,15 +122,15 @@ def response_from_wire(message: AnthropicMessage) -> Response:
     data = message.model_dump()
     blocks = data.get("content") or []
     state = [block for block in blocks if block["type"] in THINKING]
-    content = ""
-    reasoning = ""
+    content: list[str] = []
+    reasoning: list[str] = []
     calls: list[ToolCall] = []
     for block in blocks:
         kind = block.get("type")
         if kind == "text":
-            content += block.get("text", "")
+            content.append(block.get("text", ""))
         elif kind == "thinking":
-            reasoning += block.get("thinking", "")
+            reasoning.append(block.get("thinking", ""))
         elif kind == "tool_use":
             calls.append(
                 ToolCall(
@@ -161,8 +161,8 @@ def response_from_wire(message: AnthropicMessage) -> Response:
         created=0,
         model=data.get("model", ""),
         message=AssistantMessage(
-            content=content or None,
-            reasoning_content=reasoning or None,
+            content="".join(content) or None,
+            reasoning_content="".join(reasoning) or None,
             tool_calls=calls or None,
             provider_state=state or None,
         ),
@@ -220,30 +220,38 @@ class AnthropicDialect(Dialect[dict, AnthropicMessage]):
         message, then parse it."""
         message: dict = {}
         blocks: dict[int, dict] = {}
-        partial_json: dict[int, str] = {}
+        block_parts: dict[int, dict[str, list[str]]] = {}
+        partial_json: dict[int, list[str]] = {}
         for event in iter_sse(raw):
             kind = event.get("type")
             if kind == "message_start":
                 message = event.get("message") or {}
             elif kind == "content_block_start":
-                blocks[event["index"]] = dict(event.get("content_block") or {})
+                index = event["index"]
+                blocks[index] = dict(event.get("content_block") or {})
+                block_parts.pop(index, None)
             elif kind == "content_block_delta":
-                block = blocks.setdefault(event["index"], {"type": "text", "text": ""})
+                index = event["index"]
+                block = blocks.setdefault(index, {"type": "text", "text": ""})
                 delta = event.get("delta") or {}
-                if delta.get("type") == "text_delta":
-                    block["text"] = block.get("text", "") + delta.get("text", "")
-                elif delta.get("type") == "thinking_delta":
-                    block["thinking"] = block.get("thinking", "") + delta.get(
-                        "thinking", ""
-                    )
-                elif delta.get("type") == "signature_delta":
-                    block["signature"] = block.get("signature", "") + delta.get(
-                        "signature", ""
-                    )
-                elif delta.get("type") == "input_json_delta":
-                    partial_json[event["index"]] = partial_json.get(
-                        event["index"], ""
-                    ) + delta.get("partial_json", "")
+                delta_type = delta.get("type")
+                if delta_type in ("text_delta", "thinking_delta", "signature_delta"):
+                    field = delta_type.removesuffix("_delta")
+                    fields = block_parts.get(index)
+                    if fields is None:
+                        fields = {}
+                        block_parts[index] = fields
+                    parts = fields.get(field)
+                    if parts is None:
+                        parts = [block.get(field, "")]
+                        fields[field] = parts
+                    parts.append(delta.get(field, ""))
+                elif delta_type == "input_json_delta":
+                    parts = partial_json.get(index)
+                    if parts is None:
+                        parts = []
+                        partial_json[index] = parts
+                    parts.append(delta.get("partial_json", ""))
             elif kind == "message_delta":
                 message.update(
                     {
@@ -256,8 +264,11 @@ class AnthropicDialect(Dialect[dict, AnthropicMessage]):
                     **(message.get("usage") or {}),
                     **(event.get("usage") or {}),
                 }
-        for index, partial in partial_json.items():
-            blocks[index]["input"] = json.loads(partial or "{}")
+        for index, fields in block_parts.items():
+            for field, parts in fields.items():
+                blocks[index][field] = "".join(parts)
+        for index, parts in partial_json.items():
+            blocks[index]["input"] = json.loads("".join(parts) or "{}")
         message["content"] = [blocks[i] for i in sorted(blocks)]
         return response_from_wire(self.validate_response(message))
 

{verifiers-0.1.15.dev360 → verifiers-0.1.15.dev366}/verifiers/v1/dialects/base.py

@@ -13,37 +13,63 @@ the eval's model + sampling in this format's shape) and `extend` (chat-only user
 """
 
 import json
+import logging
 from abc import ABC, abstractmethod
 from collections.abc import Iterator, Mapping
 from typing import ClassVar, Generic, TypeVar
 
 from pydantic import BaseModel
+from pydantic_core import from_json
 
 from verifiers.v1.types import Messages, Response, SamplingConfig, Tool
 
 ReqT = TypeVar("ReqT")
 RespT = TypeVar("RespT", bound=BaseModel)
 
-
-def iter_sse(raw: bytes) -> list[dict]:
-    """Parse a complete SSE byte stream into its JSON data payloads, in order (skipping
-    non-JSON sentinels like OpenAI's `data: [DONE]`). Shared by the dialects' `parse_stream`."""
-    events: list[dict] = []
-    for block in raw.decode("utf-8", errors="replace").split("\n\n"):
-        data = "\n".join(
-            line.removeprefix("data:").strip()
+logger = logging.getLogger(__name__)
+
+
+def iter_sse(raw: bytes) -> Iterator[dict]:
+    """Yield JSON SSE payloads in order without retaining prior events."""
+    # Detect the wire line ending once, then scan without retaining prior payloads.
+    first_newline = raw.find(b"\n")
+    separator = (
+        b"\r\n\r\n"
+        if first_newline > 0 and raw[first_newline - 1] == ord("\r")
+        else b"\n\n"
+    )
+    start = 0
+    while start < len(raw):
+        end = raw.find(separator, start)
+        if end == -1:
+            end = len(raw)
+        block = raw[start:end]
+        data = b"\n".join(
+            line.removeprefix(b"data:").strip()
             for line in block.splitlines()
-            if line.startswith("data:")
+            if line.startswith(b"data:")
         )
-        if not data or data == "[DONE]":
-            continue
-        events.append(json.loads(data))
-    return events
+        if data and data != b"[DONE]":
+            try:
+                yield from_json(data)
+            except ValueError:
+                logger.warning(
+                    "SSE JSON fast-path failed; falling back to stdlib with invalid UTF-8 replacement"
+                )
+                yield json.loads(data.decode("utf-8", errors="replace"))
+        start = end + len(separator)
 
 
 def iter_sse_reverse(raw: bytes) -> Iterator[dict]:
     """Yield JSON SSE payloads from the end without decoding earlier events."""
-    for block in reversed(raw.decode("utf-8", errors="replace").split("\n\n")):
+    decoded = raw.decode("utf-8", errors="replace")
+    first_newline = decoded.find("\n")
+    separator = (
+        "\r\n\r\n"
+        if first_newline > 0 and decoded[first_newline - 1] == "\r"
+        else "\n\n"
+    )
+    for block in reversed(decoded.split(separator)):
         data = "\n".join(
             line.removeprefix("data:").strip()
             for line in block.splitlines()

{verifiers-0.1.15.dev360 → verifiers-0.1.15.dev366}/verifiers/v1/dialects/chat.py

@@ -229,13 +229,19 @@ class ChatDialect(Dialect[dict, ChatCompletion]):
 
     def parse_stream(self, raw: bytes) -> Response:
         """Assemble `chat.completion.chunk` deltas into one completion, then parse it."""
-        chunks = iter_sse(raw)
         message: dict = {"role": "assistant", "content": None}
+        # Join streamed fields once; repeated concatenation recopies the growing response.
+        message_parts = {key: [] for key in REASONING_FIELDS[:2] + ("content",)}
         tool_calls: dict[int, dict] = {}
+        tool_arguments: dict[int, list[str]] = {}
         reasoning_details: list[dict] = []
+        reasoning_detail_parts: dict[int, list[str]] = {}
         finish_reason = None
         usage = None
-        for chunk in chunks:
+        head = None
+        for chunk in iter_sse(raw):
+            if head is None:
+                head = chunk
             usage = chunk.get("usage") or usage
             for choice in chunk.get("choices") or []:
                 if choice.get("index", 0) != 0:
@@ -244,32 +250,43 @@ class ChatDialect(Dialect[dict, ChatCompletion]):
                 delta = choice.get("delta") or {}
                 for key in ("content", "reasoning_content", "reasoning"):
                     if delta.get(key) is not None:
-                        message[key] = (message.get(key) or "") + delta[key]
+                        message_parts[key].append(delta[key])
                 for detail in delta.get("reasoning_details") or []:
                     previous = reasoning_details[-1] if reasoning_details else {}
                     if detail.get("type") == previous.get("type") == "reasoning.text":
-                        previous["text"] = (previous.get("text") or "") + (
-                            detail.get("text") or ""
-                        )
+                        reasoning_detail_parts.setdefault(
+                            len(reasoning_details) - 1, [previous.get("text") or ""]
+                        ).append(detail.get("text") or "")
                         for field in ("signature", "format"):
                             previous[field] = previous.get(field) or detail.get(field)
                     else:
                         reasoning_details.append(detail)
                 for tc in delta.get("tool_calls") or []:
+                    index = tc.get("index", 0)
                     slot = tool_calls.setdefault(
-                        tc.get("index", 0),
+                        index,
                         {"type": "function", "function": {"name": "", "arguments": ""}},
                     )
                     slot["id"] = tc.get("id") or slot.get("id", "")
                     fn = tc.get("function") or {}
                     if fn.get("name"):
                         slot["function"]["name"] = fn["name"]
-                    slot["function"]["arguments"] += fn.get("arguments") or ""
+                    tool_arguments.setdefault(index, []).append(
+                        fn.get("arguments") or ""
+                    )
+        for key, parts in message_parts.items():
+            if parts:
+                message[key] = "".join(parts)
+        for index, parts in reasoning_detail_parts.items():
+            reasoning_details[index]["text"] = "".join(parts)
+        for index, parts in tool_arguments.items():
+            tool_calls[index]["function"]["arguments"] = "".join(parts)
         if tool_calls:
             message["tool_calls"] = [tool_calls[i] for i in sorted(tool_calls)]
         if reasoning_details:
             message["reasoning_details"] = reasoning_details
-        head = chunks[0] if chunks else {}
+        if head is None:
+            head = {}
         return response_from_wire(
             ChatCompletion.model_validate(
                 {

{verifiers-0.1.15.dev360 → verifiers-0.1.15.dev366}/verifiers/v1/graph.py

@@ -376,18 +376,20 @@ def _attribute_routed_experts(
     raw = binascii.a2b_base64(payload["data"])
     arr = np.frombuffer(raw, dtype=np.uint8).reshape(payload["shape"])
     off = path_len - int(payload.get("start", 0) or 0)
-    # The engine omits routing for the turn's final position (no forward pass follows the last
-    # generated token), so the array is one row short of the turn's new tokens. Pad the tail
-    # with a copy of the last row so the final node still aligns 1:1 with its tokens.
     needed = off + sum(len(trace.nodes[nid].token_ids) for nid in new_node_ids)
-    if arr.shape[0] and 0 < needed - arr.shape[0] <= 1:
-        arr = np.concatenate([arr, arr[-1:]], axis=0)
     for nid in new_node_ids:
         n = len(trace.nodes[nid].token_ids)
-        if n and 0 <= off and off + n <= arr.shape[0]:
+        end = off + n
+        if n and 0 <= off and end <= arr.shape[0]:
             # Own only this node's rows; a view would retain the turn's full-context array.
-            trace.nodes[nid].routed_experts = arr[off : off + n].copy()
-        off += n
+            trace.nodes[nid].routed_experts = arr[off:end].copy()
+        elif n and arr.shape[0] and 0 <= off and end == needed == arr.shape[0] + 1:
+            # The engine omits the turn's final position because no forward pass follows it.
+            # Pad only the final node's suffix instead of copying the full-context array.
+            trace.nodes[nid].routed_experts = np.concatenate(
+                [arr[off:], arr[-1:]], axis=0
+            )
+        off = end
 
 
 def _commit_turn(turn: PendingTurn, response: Response) -> None:

{verifiers-0.1.15.dev360 → verifiers-0.1.15.dev366}/verifiers/v1/interception/server.py

@@ -24,7 +24,7 @@ import contextlib
 import json
 import logging
 import secrets
-from collections.abc import Awaitable, Callable
+from collections.abc import AsyncIterator, Awaitable, Callable
 from dataclasses import dataclass, field
 from typing import TYPE_CHECKING
 
@@ -56,6 +56,7 @@ logger = logging.getLogger(__name__)
 # context window are the real limits, this is just a host-OOM backstop.
 _MAX_REQUEST_BODY = 1024**3  # 1 GiB (aiohttp's default is 1 MiB)
 _KEEPALIVE_INTERVAL_SECONDS = 3
+_STREAM_QUEUE_MAXSIZE = 16
 # The server binds loopback; callers reach it via localhost or a host tunnel (see `reachable_url`).
 _HOST = "127.0.0.1"
 
@@ -69,6 +70,20 @@ def _completion_response(completion: dict | None) -> web.Response:
     return web.Response(body=body, content_type="application/json", charset="utf-8")
 
 
+async def _queue_chunks(
+    chunks: AsyncIterator[bytes],
+    queue: asyncio.Queue[bytes | None],
+    ready: asyncio.Event,
+) -> None:
+    try:
+        async for chunk in chunks:
+            await queue.put(chunk)
+            ready.set()
+    finally:
+        await queue.put(None)
+        ready.set()
+
+
 @dataclass(frozen=True)
 class RolloutLimits:
     """Per-rollout framework limits (None = no cap), checked before each turn is served.
@@ -446,28 +461,37 @@ class InterceptionServer:
         )
         resp.content_type = reply.content_type.split(";")[0].strip()
         buffer = bytearray()
-        chunks = reply.chunks.__aiter__()
-        next_chunk = asyncio.create_task(anext(chunks, None))
+        # One bounded producer avoids per-event tasks; keepalive timeouts only cancel readiness waits.
+        queue: asyncio.Queue[bytes | None] = asyncio.Queue(
+            maxsize=_STREAM_QUEUE_MAXSIZE
+        )
+        ready = asyncio.Event()
+        producer = asyncio.create_task(_queue_chunks(reply.chunks, queue, ready))
         try:
             await resp.prepare(request)
             while True:
-                done, _ = await asyncio.wait(
-                    {next_chunk}, timeout=_KEEPALIVE_INTERVAL_SECONDS
-                )
-                if not done:
+                try:
+                    async with asyncio.timeout(_KEEPALIVE_INTERVAL_SECONDS):
+                        await ready.wait()
+                except TimeoutError:
                     await resp.write(b": keepalive\n\n")
                     continue
-                chunk = next_chunk.result()
+                chunk = queue.get_nowait()
+                if queue.empty():
+                    ready.clear()
                 if chunk is None:
+                    await producer
                     break
                 buffer += chunk
                 await resp.write(chunk)
-                next_chunk = asyncio.create_task(anext(chunks, None))
         except ConnectionResetError:
             return resp
         finally:
-            next_chunk.cancel()
-            await asyncio.gather(next_chunk, return_exceptions=True)
+            producer.cancel()
+            # Let a canceled producer enqueue EOF while unwinding.
+            if queue.full():
+                queue.get_nowait()
+            await asyncio.gather(producer, return_exceptions=True)
             await reply.close()
 
         try:

{verifiers-0.1.15.dev360 → verifiers-0.1.15.dev366}/verifiers/v1/runtimes/base.py

@@ -322,8 +322,18 @@ async def host_endpoint(port: int, is_local: bool, labels: list[str] | None = No
     try:
         yield url
     finally:
+        # Delay cancellation until the synchronous stop has finished.
+        cancelled = None
+        stop_task = asyncio.create_task(asyncio.to_thread(tunnel.sync_stop))
         with contextlib.suppress(Exception):
-            tunnel.sync_stop()
+            while not stop_task.done():
+                try:
+                    await asyncio.shield(stop_task)
+                except asyncio.CancelledError as e:
+                    cancelled = e
+            stop_task.result()
+        if cancelled is not None:
+            raise cancelled
 
 
 class _Host:

{verifiers-0.1.15.dev360 → verifiers-0.1.15.dev366}/verifiers/v1/runtimes/prime.py

@@ -7,11 +7,11 @@ direction (a program in the sandbox reaching a host service) is the shared host-
 """
 
 import asyncio
-import base64
 import contextlib
 import logging
 import shlex
-from pathlib import PurePosixPath
+import tempfile
+from pathlib import Path, PurePosixPath
 from typing import ClassVar, Literal
 
 from pydantic_config import BaseConfig
@@ -176,10 +176,22 @@ class PrimeRuntime(Runtime):
             )
 
     async def read(self, path: str) -> bytes:
-        result = await self.run(["sh", "-c", f"base64 {shlex.quote(path)}"], {})
-        if result.exit_code != 0:
-            raise SandboxError(f"read {path!r}: {result.stderr.strip()}")
-        return base64.b64decode(result.stdout)
+        # Avoid background-job log limits and base64 overhead by downloading binary data directly.
+        # The temporary file is removed on every exit, and its byte read stays off the event loop.
+        target = (
+            path
+            if path.startswith("/")
+            else f"{self.config.workdir.rstrip('/')}/{path}"
+        )
+        try:
+            with tempfile.TemporaryDirectory() as directory:
+                download = Path(directory) / "download"
+                await self._client.download_file(
+                    self._sandbox_id, target, str(download)
+                )
+                return await asyncio.to_thread(download.read_bytes)
+        except Exception as e:
+            raise SandboxError(f"read {path!r}: {e}") from e
 
     async def write(self, path: str, data: bytes) -> None:
         # Upload via the gateway (multipart) — never inline the bytes on the command line