verifiers 0.3.2.dev135__tar.gz → 0.3.2.dev137__tar.gz

{verifiers-0.3.2.dev135 → verifiers-0.3.2.dev137}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.3.2.dev135
+Version: 0.3.2.dev137
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.3.2.dev135 → verifiers-0.3.2.dev137}/tests/v1/test_e2e.py

@@ -20,7 +20,8 @@ def pair(a: str, b: str, id: str, *extra_marks):
 
 
 @pytest.mark.asyncio
-async def test_chat_harness_preserves_streamed_reasoning():
+@pytest.mark.parametrize("interruption", [None, "disconnect", "timeout", "eof"])
+async def test_chat_harness_preserves_streamed_reasoning(interruption):
     import json
 
     import httpx
@@ -61,10 +62,36 @@ async def test_chat_harness_preserves_streamed_reasoning():
     content = "".join(f"data: {json.dumps(event)}\n\n" for event in events)
     content += "data: [DONE]\n\n"
 
+    requests = []
+    streams = []
+
+    class CompletionStream(httpx.AsyncByteStream):
+        closed = False
+
+        def __init__(self, request, interrupted):
+            self.request = request
+            self.interrupted = interrupted
+
+        async def __aiter__(self):
+            if self.interrupted:
+                yield f"data: {json.dumps(chunk('discard this attempt'))}\n\n".encode()
+                if interruption == "disconnect":
+                    raise httpx.ReadError("connection reset", request=self.request)
+                if interruption == "timeout":
+                    raise httpx.ReadTimeout("read timed out", request=self.request)
+                return
+            yield content.encode()
+
+        async def aclose(self):
+            self.closed = True
+
     async def respond(request: httpx.Request) -> httpx.Response:
+        requests.append(request)
+        stream = CompletionStream(request, interruption and len(requests) == 1)
+        streams.append(stream)
         return httpx.Response(
             200,
-            content=content,
+            stream=stream,
             headers={"content-type": "text/event-stream"},
             request=request,
         )
@@ -77,6 +104,12 @@ async def test_chat_harness_preserves_streamed_reasoning():
         )
         completion = await chat(client, "test-model", [], [])
 
+    assert len(requests) == (2 if interruption else 1)
+    assert [r.headers["x-stainless-retry-count"] for r in requests] == (
+        ["0", "1"] if interruption else ["0"]
+    )
+    assert all(r.content == requests[0].content for r in requests)
+    assert all(stream.closed for stream in streams)
     message = completion.choices[0].message.model_dump(exclude_none=True)
     assert message["role"] == "assistant"
     assert message["reasoning"] == "Plan: call ls"

{verifiers-0.3.2.dev135 → verifiers-0.3.2.dev137}/verifiers/v1/harnesses/utils/core.py

@@ -3,14 +3,22 @@
 import argparse
 import asyncio
 import json
+import logging
 import subprocess
 from contextlib import AsyncExitStack
 from pathlib import Path
 from typing import TYPE_CHECKING
 
 import httpx
-from openai import APIStatusError, AsyncOpenAI, omit
+from openai import APIConnectionError, APIStatusError, AsyncOpenAI, omit
 from openai.lib.streaming.chat import AsyncChatCompletionStream
+from tenacity import (
+    AsyncRetrying,
+    before_sleep_log,
+    retry_if_exception_type,
+    stop_after_attempt,
+    wait_random_exponential,
+)
 
 if TYPE_CHECKING:
     # The harness bundles this module into the generated script before execution.
@@ -245,9 +253,28 @@ async def chat(
         kwargs["tools"] = tools
     if tools and tool_choice is not None:
         kwargs["tool_choice"] = tool_choice
-    raw_stream = await client.chat.completions.create(
-        **kwargs, stream=True, stream_options={"include_usage": True}
-    )
+    async for attempt in AsyncRetrying(
+        retry=retry_if_exception_type((APIConnectionError, httpx.TransportError)),
+        stop=stop_after_attempt(client.max_retries + 1),
+        wait=wait_random_exponential(multiplier=0.5, max=8.0),
+        before_sleep=before_sleep_log(logging.getLogger(__name__), logging.WARNING),
+        reraise=True,
+    ):
+        # Reuse the interception server's body-digest replay guard on stream retries.
+        retry_count = attempt.retry_state.attempt_number - 1
+        headers = {"x-stainless-retry-count": str(retry_count)} if retry_count else omit
+        raw_stream = await client.chat.completions.create(
+            **kwargs,
+            stream=True,
+            stream_options={"include_usage": True},
+            extra_headers=headers,
+        )
+        # The SDK retries request setup; only stream consumption is retried here.
+        with attempt:
+            return await _read_chat_completion(raw_stream)
+
+
+async def _read_chat_completion(raw_stream):
     # Accumulate native deltas without auto-parsing tool arguments or treating
     # finish_reason="length" as an exception: compaction owns that decision.
     async with AsyncChatCompletionStream(
@@ -271,7 +298,10 @@ async def chat(
             or not completion.choices
             or any(choice.finish_reason is None for choice in completion.choices)
         ):
-            raise RuntimeError("model stream ended before a completion finished")
+            raise APIConnectionError(
+                message="Model stream ended before a completion finished",
+                request=raw_stream.response.request,
+            )
         for choice in completion.choices:
             overrides = message_overrides.setdefault(choice.index, {})
             overrides.setdefault("role", "assistant")