verifiers 0.1.15.dev377__tar.gz → 0.1.15.dev380__tar.gz

{verifiers-0.1.15.dev377 → verifiers-0.1.15.dev380}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.1.15.dev377
+Version: 0.1.15.dev380
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.1.15.dev377 → verifiers-0.1.15.dev380}/verifiers/v1/cli/dashboard/eval.py

@@ -215,8 +215,8 @@ def _groups(rollouts: list[Rollout]) -> list[list[Rollout]]:
             by_task.setdefault(rollout.trace.task.idx, []).append(rollout)
     groups = list(by_task.values())
     for group in groups:
-        group.sort(key=lambda r: r.trace.timing.generation.start)
-    groups.sort(key=lambda g: g[0].trace.timing.generation.start)
+        group.sort(key=lambda r: r.trace.timing.setup.start)
+    groups.sort(key=lambda g: g[0].trace.timing.setup.start)
     return groups
 
 
@@ -253,11 +253,14 @@ def Rows(groups: list[list[Rollout]], now: float, runtime_type: str) -> Table:
             )
             runtime = f"{runtime_type}({descriptor})" if descriptor else runtime_type
             turns = t.num_turns
-            start = t.timing.generation.start
+            start = t.timing.setup.start
             end = (
                 t.timing.scoring.end
                 or t.timing.finalize.end
                 or t.timing.generation.end
+                # a rollout that errored in setup has only setup.end — freeze there once done,
+                # else (still running) the timer would grow off `now` forever
+                or (t.timing.setup.end if t.is_completed else 0)
                 or now
             )
             prompt, completion, cached, reasoning, nbranches = _tokens(t)

{verifiers-0.1.15.dev377 → verifiers-0.1.15.dev380}/verifiers/v1/clients/eval.py

@@ -56,7 +56,8 @@ _BLOCKED_REQUEST_HEADERS = frozenset(
         "signature-input",
     }
 )
-_SSE_EVENT_END = re.compile(rb"(?:\r\n|\r|\n){2}")
+# Atomic so one CRLF cannot backtrack into two line endings and split an event mid-field.
+_SSE_EVENT_END = re.compile(rb"(?>\r\n|\r|\n){2}")
 
 
 class EvalClient(Client):

{verifiers-0.1.15.dev377 → verifiers-0.1.15.dev380}/verifiers/v1/dialects/__init__.py

@@ -6,7 +6,7 @@ resolved from the endpoint the program's SDK posts to — no per-harness declara
 """
 
 from verifiers.v1.dialects.anthropic import AnthropicDialect
-from verifiers.v1.dialects.base import Dialect, iter_sse
+from verifiers.v1.dialects.base import Dialect, StreamParser, iter_sse
 from verifiers.v1.dialects.chat import (
     FINISH_REASONS,
     ChatDialect,
@@ -27,6 +27,7 @@ __all__ = [
     "AnthropicDialect",
     "ChatDialect",
     "ResponsesDialect",
+    "StreamParser",
     "iter_sse",
     "parse_message",
     "parse_tools",

{verifiers-0.1.15.dev377 → verifiers-0.1.15.dev380}/verifiers/v1/dialects/anthropic.py

@@ -7,11 +7,12 @@ trace. `count_tokens` is relayed verbatim (an `aux_route`), never recorded.
 """
 
 import json
-from collections.abc import Mapping
+from collections.abc import Callable, Mapping
+from dataclasses import dataclass, field
 
 from anthropic.types import Message as AnthropicMessage
 
-from verifiers.v1.dialects.base import Dialect, iter_sse
+from verifiers.v1.dialects.base import Dialect, StreamParser, parse_sse_event
 from verifiers.v1.types import (
     AssistantMessage,
     ContentPart,
@@ -171,6 +172,76 @@ def response_from_wire(message: AnthropicMessage) -> Response:
     )
 
 
+@dataclass
+class AnthropicStreamParser(StreamParser):
+    """Incrementally assemble Anthropic message events without retaining SSE bytes."""
+
+    validate_response: Callable[[dict], AnthropicMessage]
+    message: dict = field(default_factory=dict)
+    blocks: dict[int, dict] = field(default_factory=dict)
+    block_parts: dict[int, dict[str, list[str]]] = field(default_factory=dict)
+    partial_json: dict[int, list[str]] = field(default_factory=dict)
+
+    def feed(self, raw: bytes) -> None:
+        event = parse_sse_event(raw)
+        if event is None:
+            return
+        kind = event.get("type")
+        if kind == "message_start":
+            self.message = event.get("message") or {}
+        elif kind == "content_block_start":
+            index = event["index"]
+            self.blocks[index] = dict(event.get("content_block") or {})
+            self.block_parts.pop(index, None)
+        elif kind == "content_block_delta":
+            index = event["index"]
+            block = self.blocks.setdefault(index, {"type": "text", "text": ""})
+            delta = event.get("delta") or {}
+            delta_type = delta.get("type")
+            if delta_type in (
+                "text_delta",
+                "thinking_delta",
+                "signature_delta",
+            ):
+                field_name = delta_type.removesuffix("_delta")
+                fields = self.block_parts.get(index)
+                if fields is None:
+                    fields = {}
+                    self.block_parts[index] = fields
+                parts = fields.get(field_name)
+                if parts is None:
+                    parts = [block.get(field_name, "")]
+                    fields[field_name] = parts
+                parts.append(delta.get(field_name, ""))
+            elif delta_type == "input_json_delta":
+                parts = self.partial_json.get(index)
+                if parts is None:
+                    parts = []
+                    self.partial_json[index] = parts
+                parts.append(delta.get("partial_json", ""))
+        elif kind == "message_delta":
+            self.message.update(
+                {
+                    key: value
+                    for key, value in (event.get("delta") or {}).items()
+                    if value is not None
+                }
+            )
+            self.message["usage"] = {
+                **(self.message.get("usage") or {}),
+                **(event.get("usage") or {}),
+            }
+
+    def finish(self) -> Response:
+        for index, fields in self.block_parts.items():
+            for field_name, parts in fields.items():
+                self.blocks[index][field_name] = "".join(parts)
+        for index, parts in self.partial_json.items():
+            self.blocks[index]["input"] = json.loads("".join(parts) or "{}")
+        self.message["content"] = [self.blocks[index] for index in sorted(self.blocks)]
+        return response_from_wire(self.validate_response(self.message))
+
+
 class AnthropicDialect(Dialect[dict, AnthropicMessage]):
     """The Anthropic Messages wire format."""
 
@@ -215,62 +286,8 @@ class AnthropicDialect(Dialect[dict, AnthropicMessage]):
             raw["usage"].pop("service_tier")
         return super().validate_response(raw)
 
-    def parse_stream(self, raw: bytes) -> Response:
-        """Assemble message_start / content_block_* / message_delta events into the complete
-        message, then parse it."""
-        message: dict = {}
-        blocks: dict[int, dict] = {}
-        block_parts: dict[int, dict[str, list[str]]] = {}
-        partial_json: dict[int, list[str]] = {}
-        for event in iter_sse(raw):
-            kind = event.get("type")
-            if kind == "message_start":
-                message = event.get("message") or {}
-            elif kind == "content_block_start":
-                index = event["index"]
-                blocks[index] = dict(event.get("content_block") or {})
-                block_parts.pop(index, None)
-            elif kind == "content_block_delta":
-                index = event["index"]
-                block = blocks.setdefault(index, {"type": "text", "text": ""})
-                delta = event.get("delta") or {}
-                delta_type = delta.get("type")
-                if delta_type in ("text_delta", "thinking_delta", "signature_delta"):
-                    field = delta_type.removesuffix("_delta")
-                    fields = block_parts.get(index)
-                    if fields is None:
-                        fields = {}
-                        block_parts[index] = fields
-                    parts = fields.get(field)
-                    if parts is None:
-                        parts = [block.get(field, "")]
-                        fields[field] = parts
-                    parts.append(delta.get(field, ""))
-                elif delta_type == "input_json_delta":
-                    parts = partial_json.get(index)
-                    if parts is None:
-                        parts = []
-                        partial_json[index] = parts
-                    parts.append(delta.get("partial_json", ""))
-            elif kind == "message_delta":
-                message.update(
-                    {
-                        k: v
-                        for k, v in (event.get("delta") or {}).items()
-                        if v is not None
-                    }
-                )
-                message["usage"] = {
-                    **(message.get("usage") or {}),
-                    **(event.get("usage") or {}),
-                }
-        for index, fields in block_parts.items():
-            for field, parts in fields.items():
-                blocks[index][field] = "".join(parts)
-        for index, parts in partial_json.items():
-            blocks[index]["input"] = json.loads("".join(parts) or "{}")
-        message["content"] = [blocks[i] for i in sorted(blocks)]
-        return response_from_wire(self.validate_response(message))
+    def stream_parser(self) -> StreamParser:
+        return AnthropicStreamParser(self.validate_response)
 
     def apply_overrides(self, body: dict, model: str, sampling: SamplingConfig) -> dict:
         # Forward verbatim except the eval's model + sampling. `temperature`/`top_p` are

{verifiers-0.1.15.dev377 → verifiers-0.1.15.dev380}/verifiers/v1/dialects/base.py

@@ -5,17 +5,17 @@ trace from the program's native request + the provider's native response. The se
 every registered dialect's `routes` (see `dialects.DIALECTS`), so a request's format is resolved
 from the endpoint the program's SDK posts to — the harness declares nothing.
 
-The eval client relays a request's bytes verbatim to a matching endpoint, so the dialect only
-parses a *copy* for the trace (incl. assembling a relayed SSE stream via `parse_stream`); the
-renderer is chat-only. A dialect is therefore mostly wire -> vf
-(`parse_request`/`parse_response`/`parse_stream`); the exceptions are `apply_overrides` (impose
-the eval's model + sampling in this format's shape) and `extend` (chat-only user-sim injection).
+The eval client relays a request's bytes verbatim to a matching endpoint, while a dialect-owned
+`StreamParser` incrementally assembles a response copy for the trace; the renderer is chat-only.
+A dialect is therefore mostly wire -> vf (`parse_request`/`parse_response`/`stream_parser`); the
+exceptions are `apply_overrides` (impose the eval's model + sampling in this format's shape) and
+`extend` (chat-only user-sim injection).
 """
 
 import json
 import logging
 from abc import ABC, abstractmethod
-from collections.abc import Iterator, Mapping
+from collections.abc import Callable, Iterator, Mapping
 from typing import ClassVar, Generic, TypeVar
 
 from pydantic import BaseModel
@@ -29,6 +29,37 @@ RespT = TypeVar("RespT", bound=BaseModel)
 logger = logging.getLogger(__name__)
 
 
+def is_sse_done_event(raw: bytes) -> bool:
+    """Whether one complete SSE event carries the DONE sentinel."""
+    # Ordinary OpenAI events carry JSON objects; reject their hot path before splitting lines.
+    if raw.startswith((b"data: {", b"data:{")):
+        return False
+    data = b"\n".join(
+        line.removeprefix(b"data:").strip()
+        for line in raw.splitlines()
+        if line.startswith(b"data:")
+    )
+    return data == b"[DONE]"
+
+
+def parse_sse_event(raw: bytes) -> dict | None:
+    """Parse one complete SSE event's JSON data payload, ignoring comments and sentinels."""
+    data = b"\n".join(
+        line.removeprefix(b"data:").strip()
+        for line in raw.splitlines()
+        if line.startswith(b"data:")
+    )
+    if not data or data == b"[DONE]":
+        return None
+    try:
+        return from_json(data)
+    except ValueError:
+        logger.warning(
+            "SSE JSON fast-path failed; falling back to stdlib with invalid UTF-8 replacement"
+        )
+        return json.loads(data.decode("utf-8", errors="replace"))
+
+
 def iter_sse(raw: bytes) -> Iterator[dict]:
     """Yield JSON SSE payloads in order without retaining prior events."""
     # Detect the wire line ending once, then scan without retaining prior payloads.
@@ -44,19 +75,9 @@ def iter_sse(raw: bytes) -> Iterator[dict]:
         if end == -1:
             end = len(raw)
         block = raw[start:end]
-        data = b"\n".join(
-            line.removeprefix(b"data:").strip()
-            for line in block.splitlines()
-            if line.startswith(b"data:")
-        )
-        if data and data != b"[DONE]":
-            try:
-                yield from_json(data)
-            except ValueError:
-                logger.warning(
-                    "SSE JSON fast-path failed; falling back to stdlib with invalid UTF-8 replacement"
-                )
-                yield json.loads(data.decode("utf-8", errors="replace"))
+        event = parse_sse_event(block)
+        if event is not None:
+            yield event
         start = end + len(separator)
 
 
@@ -80,6 +101,20 @@ def iter_sse_reverse(raw: bytes) -> Iterator[dict]:
         yield json.loads(data)
 
 
+class StreamParser(ABC):
+    """Incrementally assemble one native SSE stream into a vf response."""
+
+    feed: Callable[[bytes], None]
+    """Consume one complete SSE event without retaining its raw bytes."""
+
+    on_done: Callable[[], None] | None = None
+    """Preserve terminal state before events following the DONE sentinel."""
+
+    @abstractmethod
+    def finish(self) -> Response:
+        """Finalize and return the assembled response after the stream ends."""
+
+
 class Dialect(ABC, Generic[ReqT, RespT]):
     """One native API's wire format, fully typed over its request (`ReqT`) and response
     (`RespT`). The single place a protocol lives: implement a `Dialect` + register it in
@@ -133,9 +168,8 @@ class Dialect(ABC, Generic[ReqT, RespT]):
         return self.response_type.model_validate(raw)
 
     @abstractmethod
-    def parse_stream(self, raw: bytes) -> Response:
-        """A complete native SSE byte stream -> the vf `Response` (assembling the final message
-        from the events), to record a relayed stream on the trace."""
+    def stream_parser(self) -> StreamParser:
+        """Create the per-request incremental parser for a native SSE response."""
 
     @abstractmethod
     def apply_overrides(self, body: ReqT, model: str, sampling: SamplingConfig) -> ReqT:

{verifiers-0.1.15.dev377 → verifiers-0.1.15.dev380}/verifiers/v1/dialects/chat.py

@@ -8,11 +8,12 @@ read them in the same precedence (`reasoning` / `reasoning_content` / `reasoning
 
 import time
 from collections.abc import Mapping
+from dataclasses import dataclass, field as dataclass_field
 from typing import Any
 
 from openai.types.chat import ChatCompletion
 
-from verifiers.v1.dialects.base import Dialect, iter_sse
+from verifiers.v1.dialects.base import Dialect, StreamParser, parse_sse_event
 from verifiers.v1.types import (
     AssistantMessage,
     FinishReason,
@@ -202,6 +203,101 @@ def response_from_wire(completion: ChatCompletion) -> Response:
     )
 
 
+@dataclass
+class ChatStreamParser(StreamParser):
+    """Incrementally assemble Chat Completions deltas without retaining SSE bytes."""
+
+    message: dict = dataclass_field(
+        default_factory=lambda: {"role": "assistant", "content": None}
+    )
+    message_parts: dict[str, list[str]] = dataclass_field(
+        default_factory=lambda: {key: [] for key in REASONING_FIELDS[:2] + ("content",)}
+    )
+    tool_calls: dict[int, dict] = dataclass_field(default_factory=dict)
+    tool_arguments: dict[int, list[str]] = dataclass_field(default_factory=dict)
+    reasoning_details: list[dict] = dataclass_field(default_factory=list)
+    reasoning_detail_parts: dict[int, list[str]] = dataclass_field(default_factory=dict)
+    finish_reason: str | None = None
+    usage: dict | None = None
+    head: dict | None = None
+
+    def feed(self, raw: bytes) -> None:
+        chunk = parse_sse_event(raw)
+        if chunk is None:
+            return
+        if self.head is None:
+            self.head = chunk
+        self.usage = chunk.get("usage") or self.usage
+        for choice in chunk.get("choices") or []:
+            if choice.get("index", 0) != 0:
+                continue
+            self.finish_reason = choice.get("finish_reason") or self.finish_reason
+            delta = choice.get("delta") or {}
+            for key in ("content", "reasoning_content", "reasoning"):
+                if delta.get(key) is not None:
+                    self.message_parts[key].append(delta[key])
+            for detail in delta.get("reasoning_details") or []:
+                previous = self.reasoning_details[-1] if self.reasoning_details else {}
+                if detail.get("type") == previous.get("type") == "reasoning.text":
+                    self.reasoning_detail_parts.setdefault(
+                        len(self.reasoning_details) - 1,
+                        [previous.get("text") or ""],
+                    ).append(detail.get("text") or "")
+                    for field_name in ("signature", "format"):
+                        previous[field_name] = previous.get(field_name) or detail.get(
+                            field_name
+                        )
+                else:
+                    self.reasoning_details.append(detail)
+            for tool_call in delta.get("tool_calls") or []:
+                index = tool_call.get("index", 0)
+                slot = self.tool_calls.setdefault(
+                    index,
+                    {"type": "function", "function": {"name": "", "arguments": ""}},
+                )
+                slot["id"] = tool_call.get("id") or slot.get("id", "")
+                function = tool_call.get("function") or {}
+                if function.get("name"):
+                    slot["function"]["name"] = function["name"]
+                self.tool_arguments.setdefault(index, []).append(
+                    function.get("arguments") or ""
+                )
+
+    def finish(self) -> Response:
+        for key, parts in self.message_parts.items():
+            if parts:
+                self.message[key] = "".join(parts)
+        for index, parts in self.reasoning_detail_parts.items():
+            self.reasoning_details[index]["text"] = "".join(parts)
+        for index, parts in self.tool_arguments.items():
+            self.tool_calls[index]["function"]["arguments"] = "".join(parts)
+        if self.tool_calls:
+            self.message["tool_calls"] = [
+                self.tool_calls[index] for index in sorted(self.tool_calls)
+            ]
+        if self.reasoning_details:
+            self.message["reasoning_details"] = self.reasoning_details
+        head = self.head or {}
+        return response_from_wire(
+            ChatCompletion.model_validate(
+                {
+                    "id": head.get("id", "vf-intercept"),
+                    "object": "chat.completion",
+                    "created": head.get("created", int(time.time())),
+                    "model": head.get("model", ""),
+                    "choices": [
+                        {
+                            "index": 0,
+                            "message": self.message,
+                            "finish_reason": self.finish_reason or "stop",
+                        }
+                    ],
+                    "usage": self.usage,
+                }
+            )
+        )
+
+
 class ChatDialect(Dialect[dict, ChatCompletion]):
     """The OpenAI chat-completions wire format."""
 
@@ -227,84 +323,8 @@ class ChatDialect(Dialect[dict, ChatCompletion]):
     def parse_response(self, response: ChatCompletion) -> Response:
         return response_from_wire(response)
 
-    def parse_stream(self, raw: bytes) -> Response:
-        """Assemble `chat.completion.chunk` deltas into one completion, then parse it."""
-        message: dict = {"role": "assistant", "content": None}
-        # Join streamed fields once; repeated concatenation recopies the growing response.
-        message_parts = {key: [] for key in REASONING_FIELDS[:2] + ("content",)}
-        tool_calls: dict[int, dict] = {}
-        tool_arguments: dict[int, list[str]] = {}
-        reasoning_details: list[dict] = []
-        reasoning_detail_parts: dict[int, list[str]] = {}
-        finish_reason = None
-        usage = None
-        head = None
-        for chunk in iter_sse(raw):
-            if head is None:
-                head = chunk
-            usage = chunk.get("usage") or usage
-            for choice in chunk.get("choices") or []:
-                if choice.get("index", 0) != 0:
-                    continue
-                finish_reason = choice.get("finish_reason") or finish_reason
-                delta = choice.get("delta") or {}
-                for key in ("content", "reasoning_content", "reasoning"):
-                    if delta.get(key) is not None:
-                        message_parts[key].append(delta[key])
-                for detail in delta.get("reasoning_details") or []:
-                    previous = reasoning_details[-1] if reasoning_details else {}
-                    if detail.get("type") == previous.get("type") == "reasoning.text":
-                        reasoning_detail_parts.setdefault(
-                            len(reasoning_details) - 1, [previous.get("text") or ""]
-                        ).append(detail.get("text") or "")
-                        for field in ("signature", "format"):
-                            previous[field] = previous.get(field) or detail.get(field)
-                    else:
-                        reasoning_details.append(detail)
-                for tc in delta.get("tool_calls") or []:
-                    index = tc.get("index", 0)
-                    slot = tool_calls.setdefault(
-                        index,
-                        {"type": "function", "function": {"name": "", "arguments": ""}},
-                    )
-                    slot["id"] = tc.get("id") or slot.get("id", "")
-                    fn = tc.get("function") or {}
-                    if fn.get("name"):
-                        slot["function"]["name"] = fn["name"]
-                    tool_arguments.setdefault(index, []).append(
-                        fn.get("arguments") or ""
-                    )
-        for key, parts in message_parts.items():
-            if parts:
-                message[key] = "".join(parts)
-        for index, parts in reasoning_detail_parts.items():
-            reasoning_details[index]["text"] = "".join(parts)
-        for index, parts in tool_arguments.items():
-            tool_calls[index]["function"]["arguments"] = "".join(parts)
-        if tool_calls:
-            message["tool_calls"] = [tool_calls[i] for i in sorted(tool_calls)]
-        if reasoning_details:
-            message["reasoning_details"] = reasoning_details
-        if head is None:
-            head = {}
-        return response_from_wire(
-            ChatCompletion.model_validate(
-                {
-                    "id": head.get("id", "vf-intercept"),
-                    "object": "chat.completion",
-                    "created": head.get("created", int(time.time())),
-                    "model": head.get("model", ""),
-                    "choices": [
-                        {
-                            "index": 0,
-                            "message": message,
-                            "finish_reason": finish_reason or "stop",
-                        }
-                    ],
-                    "usage": usage,
-                }
-            )
-        )
+    def stream_parser(self) -> StreamParser:
+        return ChatStreamParser()
 
     def apply_overrides(self, body: dict, model: str, sampling: SamplingConfig) -> dict:
         # Forward the program's body verbatim, overlaying only what the eval owns: the model and

{verifiers-0.1.15.dev377 → verifiers-0.1.15.dev380}/verifiers/v1/dialects/responses.py

@@ -8,6 +8,7 @@ endpoint and this dialect parses a copy for the trace. Server-side statefulness
 """
 
 import json
+from collections import deque
 from typing import Any, cast
 
 from openai.types.responses import (
@@ -26,7 +27,7 @@ from openai.types.responses.response_usage import (
 )
 from pydantic import BaseModel, ConfigDict
 
-from verifiers.v1.dialects.base import Dialect, iter_sse_reverse
+from verifiers.v1.dialects.base import Dialect, StreamParser, iter_sse_reverse
 from verifiers.v1.types import (
     AssistantMessage,
     ContentPart,
@@ -233,6 +234,28 @@ def response_from_wire(response: OpenAIResponse) -> Response:
     )
 
 
+class ResponsesStreamParser(StreamParser):
+    """Retain only the complete terminal response event and trailing DONE event."""
+
+    def __init__(self) -> None:
+        self.events: deque[bytes] = deque(maxlen=2)
+        self.feed = self.events.append
+        self.terminal_events: tuple[bytes, ...] | None = None
+
+    def on_done(self) -> None:
+        # Freeze the terminal tail before later relay chunks can evict it.
+        self.terminal_events = tuple(self.events)
+
+    def finish(self) -> Response:
+        events = self.terminal_events or self.events
+        for event in iter_sse_reverse(b"".join(events)):
+            if event.get("type") in FINAL_EVENTS:
+                return response_from_wire(
+                    OpenAIResponse.model_validate(event["response"])
+                )
+        raise ValueError("Responses stream ended without a terminal event")
+
+
 class ResponsesDialect(Dialect[dict, OpenAIResponse]):
     """The OpenAI Responses wire format."""
 
@@ -295,15 +318,8 @@ class ResponsesDialect(Dialect[dict, OpenAIResponse]):
     def parse_response(self, response: OpenAIResponse) -> Response:
         return response_from_wire(response)
 
-    def parse_stream(self, raw: bytes) -> Response:
-        """The terminal event (`response.completed` / `.incomplete` / `.failed`) carries the
-        full response object — parse that."""
-        for event in iter_sse_reverse(raw):
-            if event.get("type") in FINAL_EVENTS:
-                return response_from_wire(
-                    OpenAIResponse.model_validate(event["response"])
-                )
-        raise ValueError("Responses stream ended without a terminal event")
+    def stream_parser(self) -> StreamParser:
+        return ResponsesStreamParser()
 
     def apply_overrides(self, body: dict, model: str, sampling: SamplingConfig) -> dict:
         # Forward verbatim except the eval's model + sampling, mapped to the Responses shape

{verifiers-0.1.15.dev377 → verifiers-0.1.15.dev380}/verifiers/v1/interception/server.py

@@ -34,6 +34,7 @@ from pydantic_core import PydanticSerializationError, from_json, to_json
 
 from verifiers.v1.clients import RolloutContext
 from verifiers.v1.dialects import DIALECTS, Dialect
+from verifiers.v1.dialects.base import is_sse_done_event
 from verifiers.v1 import graph
 from verifiers.v1.errors import (
     OverlongPromptError,
@@ -411,8 +412,8 @@ class InterceptionServer:
         prompt: Messages,
     ) -> web.StreamResponse:
         """A streamed (SSE) model turn: relay the provider's stream through to the program,
-        accumulating the bytes to record the turn on the trace. Single-shot — a streamed turn
-        never drives a user simulator (the only client that streams is the eval relay)."""
+        incrementally assembling the response to record on the trace. Single-shot — a streamed
+        turn never drives a user simulator (the only client that streams is the eval relay)."""
         try:
             refused = await session.refused()
         except RolloutError as e:
@@ -460,13 +461,17 @@ class InterceptionServer:
             headers={"Cache-Control": "no-cache", "Connection": "keep-alive"}
         )
         resp.content_type = reply.content_type.split(";")[0].strip()
-        buffer = bytearray()
+        # Parse complete events as they relay, avoiding a full-stream byte copy.
+        parser = dialect.stream_parser()
+        feed_event = parser.feed
+        on_done = parser.on_done
         # One bounded producer avoids per-event tasks; keepalive timeouts only cancel readiness waits.
         queue: asyncio.Queue[bytes | None] = asyncio.Queue(
             maxsize=_STREAM_QUEUE_MAXSIZE
         )
         ready = asyncio.Event()
         producer = asyncio.create_task(_queue_chunks(reply.chunks, queue, ready))
+        parser_error: Exception | None = None
         try:
             await resp.prepare(request)
             while True:
@@ -482,8 +487,14 @@ class InterceptionServer:
                 if chunk is None:
                     await producer
                     break
-                buffer += chunk
                 await resp.write(chunk)
+                if parser_error is None:
+                    try:
+                        if on_done is not None and is_sse_done_event(chunk):
+                            on_done()
+                        feed_event(chunk)
+                    except Exception as e:
+                        parser_error = e
         except ConnectionResetError:
             return resp
         finally:
@@ -495,7 +506,9 @@ class InterceptionServer:
             await reply.close()
 
         try:
-            turn.commit(dialect.parse_stream(bytes(buffer)))
+            if parser_error is not None:
+                raise parser_error
+            turn.commit(parser.finish())
             logger.debug("intercept stream turn: id=%s", session.trace.id)
         finally:
             with contextlib.suppress(ConnectionResetError):