verifiers 0.1.13.dev2__tar.gz → 0.1.13.dev3__tar.gz

{verifiers-0.1.13.dev2 → verifiers-0.1.13.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.1.13.dev2
+Version: 0.1.13.dev3
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

verifiers-0.1.13.dev3/tests/test_interception_utils.py

@@ -0,0 +1,133 @@
+import asyncio
+from unittest.mock import AsyncMock, MagicMock
+
+from verifiers.errors import InfraError
+from verifiers.types import (
+    Response,
+    ResponseMessage,
+    TextContentPart,
+    ToolCall,
+    Usage,
+)
+from verifiers.utils import interception_utils
+from verifiers.utils.interception_utils import (
+    InterceptionServer,
+    StreamInterrupted,
+    create_empty_completion,
+    serialize_intercept_response,
+)
+
+
+def test_serialize_intercept_response_from_vf_response_uses_chat_completion_shape():
+    response = Response(
+        id="resp_1",
+        created=123,
+        model="test-model",
+        usage=Usage(
+            prompt_tokens=10,
+            reasoning_tokens=0,
+            completion_tokens=5,
+            total_tokens=15,
+        ),
+        message=ResponseMessage(
+            content=[TextContentPart(text="hello "), {"type": "text", "text": "world"}],
+            reasoning_content=None,
+            tool_calls=[
+                ToolCall(id="call_1", name="echo", arguments='{"x": 1}'),
+            ],
+            finish_reason="tool_calls",
+            is_truncated=False,
+            tokens=None,
+        ),
+    )
+
+    payload = serialize_intercept_response(response)
+
+    assert payload["id"] == "resp_1"
+    assert payload["object"] == "chat.completion"
+    assert payload["model"] == "test-model"
+    assert payload["choices"][0]["message"]["role"] == "assistant"
+    assert payload["choices"][0]["message"]["content"] == "hello world"
+    assert payload["choices"][0]["message"]["tool_calls"] == [
+        {
+            "id": "call_1",
+            "type": "function",
+            "function": {"name": "echo", "arguments": '{"x": 1}'},
+        }
+    ]
+    assert payload["choices"][0]["finish_reason"] == "tool_calls"
+    assert payload["usage"]["prompt_tokens"] == 10
+    assert payload["usage"]["completion_tokens"] == 5
+    assert payload["usage"]["total_tokens"] == 15
+
+
+def test_serialize_intercept_response_passthrough_native_chat_completion():
+    native = create_empty_completion("native-model")
+    payload = serialize_intercept_response(native)
+
+    assert payload["object"] == "chat.completion"
+    assert payload["model"] == "native-model"
+    assert len(payload["choices"]) == 1
+
+
+def test_set_rollout_error_attaches_stream_interrupted_to_state():
+    server = InterceptionServer(port=0)
+    state: dict = {}
+    server.register_rollout("r1", state=state)
+
+    err = StreamInterrupted("tunnel died")
+    server._set_rollout_error("r1", err)
+
+    assert state["error"] is err
+    assert isinstance(state["error"], InfraError)
+
+
+def test_set_rollout_error_does_not_clobber_existing_error():
+    # First error wins — later write failures must not hide the original cause.
+    server = InterceptionServer(port=0)
+    original = InfraError("original")
+    state: dict = {"error": original}
+    server.register_rollout("r1", state=state)
+
+    server._set_rollout_error("r1", StreamInterrupted("later"))
+
+    assert state["error"] is original
+
+
+async def test_streaming_write_failure_surfaces_to_state(monkeypatch):
+    """The real failure path: a mid-SSE transport close on the client side
+    raises out of ``response.write(...)``. The except branch must funnel
+    that into ``state["error"]`` so the rollout halts via ``has_error``."""
+    server = InterceptionServer(port=0)
+    state: dict = {}
+    server.register_rollout("r1", state=state)
+
+    # Mock StreamResponse whose second write raises (first write succeeds
+    # to prove we're in the streaming loop, not failing at prepare()).
+    writes: list[bytes] = []
+
+    async def fake_write(data: bytes) -> None:
+        writes.append(data)
+        if len(writes) >= 2:
+            raise ConnectionResetError("client closed transport")
+
+    fake_response = MagicMock()
+    fake_response.prepare = AsyncMock()
+    fake_response.write = AsyncMock(side_effect=fake_write)
+    fake_response.write_eof = AsyncMock()
+    monkeypatch.setattr(
+        interception_utils.web, "StreamResponse", lambda **_: fake_response
+    )
+
+    chunk_queue: asyncio.Queue = asyncio.Queue()
+    await chunk_queue.put({"choices": [{"delta": {"content": "hi"}}]})
+    await chunk_queue.put({"choices": [{"delta": {"content": " there"}}]})
+    intercept = {
+        "chunk_queue": chunk_queue,
+        "response_future": asyncio.Future(),
+    }
+
+    await server._handle_streaming_response(MagicMock(), "r1", intercept)
+
+    assert isinstance(state["error"], StreamInterrupted)
+    assert "ConnectionResetError" in str(state["error"])

{verifiers-0.1.13.dev2 → verifiers-0.1.13.dev3}/verifiers/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.1.13.dev2"
+__version__ = "0.1.13.dev3"
 
 import importlib
 import os

{verifiers-0.1.13.dev2 → verifiers-0.1.13.dev3}/verifiers/envs/experimental/cli_agent_env.py

@@ -248,8 +248,11 @@ class CliAgentEnv(SandboxMixin, vf.MultiTurnEnv):
         )
         await self.create_sandbox(state, sandbox_request)
 
-        # Register rollout for interception
-        request_id_queue = interception_server.register_rollout(rollout_id)
+        # Register rollout for interception. Pass state so the server can
+        # surface stream-interruption errors (e.g. tunnel dies mid-SSE) back
+        # onto the rollout; without this the agent sees a truncated stream
+        # and often exits with code 0 and an empty trajectory.
+        request_id_queue = interception_server.register_rollout(rollout_id, state=state)
         state["request_id_queue"] = request_id_queue
         state["agent_completed"] = False
 

{verifiers-0.1.13.dev2 → verifiers-0.1.13.dev3}/verifiers/envs/experimental/composable/harnesses/rlm.py

@@ -259,18 +259,21 @@ def rlm_harness(
     local_checkout: str | Path | None = None,
     gh_token: str | None = None,
 ) -> Harness:
-    resolved_local_checkout = resolve_local_checkout(
-        local_checkout,
-        rlm_repo_url=rlm_repo_url,
-        rlm_branch=rlm_branch,
-        gh_token=gh_token,
-    )
-    upload_dirs: dict[str, Traversable | Path] = {
-        DEFAULT_RLM_CHECKOUT_UPLOAD_NAME: resolved_local_checkout,
-    }
     upload_dir_mapping: dict[str, str] = {
         DEFAULT_RLM_CHECKOUT_UPLOAD_NAME: DEFAULT_RLM_CHECKOUT_PATH,
     }
+
+    def get_upload_dirs() -> dict[str, Traversable | Path]:
+        resolved_local_checkout = resolve_local_checkout(
+            local_checkout,
+            rlm_repo_url=rlm_repo_url,
+            rlm_branch=rlm_branch,
+            gh_token=gh_token,
+        )
+        return {
+            DEFAULT_RLM_CHECKOUT_UPLOAD_NAME: resolved_local_checkout,
+        }
+
     return Harness(
         install_script=build_install_script(),
         run_command=build_run_command(instruction_path, workdir),
@@ -278,7 +281,7 @@ def rlm_harness(
         system_prompt_path=DEFAULT_APPEND_TO_SYSTEM_PROMPT_PATH,
         instruction_path=instruction_path,
         skills_path="/task/rlm-skills",
-        get_upload_dirs=lambda: upload_dirs,
+        get_upload_dirs=get_upload_dirs,
         upload_dir_mapping=upload_dir_mapping,
         metrics_path="{workdir}/.rlm/sessions/*/meta.json",
         metrics_key="metrics",

{verifiers-0.1.13.dev2 → verifiers-0.1.13.dev3}/verifiers/utils/interception_utils.py

@@ -23,12 +23,23 @@ from openai.types.chat.chat_completion_chunk import (
     Choice as ChunkChoice,
 )
 
+from verifiers.errors import InfraError
 from verifiers.types import Response
 from verifiers.utils.logging_utils import truncate
 
 logger = logging.getLogger(__name__)
 
 
+class StreamInterrupted(InfraError):
+    """Raised when the intercepted streaming response to the agent is cut short.
+
+    Without this, a mid-stream transport failure would be swallowed here and
+    the agent would observe a truncated (but syntactically valid) SSE stream,
+    often exiting with code 0 and an empty trajectory — bypassing the
+    non-zero-exit error capture in `CliAgentEnv.poll_job_completion`.
+    """
+
+
 class InterceptionServer:
     """
     HTTP server that intercepts API requests from agents.
@@ -99,10 +110,26 @@ class InterceptionServer:
                     self._site = None
                     self._app = None
 
-    def register_rollout(self, rollout_id: str) -> asyncio.Queue:
+    def _set_rollout_error(self, rollout_id: str, error: BaseException) -> None:
+        """Attach `error` to the rollout's state if one is registered and
+        unset. First error wins — later failures (e.g. the downstream
+        `response_future` raising too) should not clobber the original cause.
+        """
+        context = self.active_rollouts.get(rollout_id)
+        if context is None:
+            return
+        state = context.get("state")
+        if state is None or state.get("error"):
+            return
+        state["error"] = error
+
+    def register_rollout(
+        self, rollout_id: str, state: dict[str, Any] | None = None
+    ) -> asyncio.Queue:
         request_queue: asyncio.Queue = asyncio.Queue()
         self.active_rollouts[rollout_id] = {
             "request_id_queue": request_queue,
+            "state": state,
         }
         return request_queue
 
@@ -214,6 +241,12 @@ class InterceptionServer:
             logger.debug(f"[{rollout_id}] Streaming cancelled")
         except Exception as e:
             logger.error(f"[{rollout_id}] Streaming error: {e}")
+            self._set_rollout_error(
+                rollout_id,
+                StreamInterrupted(
+                    f"Interception stream to agent interrupted: {type(e).__name__}: {e}"
+                ),
+            )
             return response
 
         try:

verifiers-0.1.13.dev2/tests/test_interception_utils.py

@@ -1,63 +0,0 @@
-from verifiers.types import (
-    Response,
-    ResponseMessage,
-    TextContentPart,
-    ToolCall,
-    Usage,
-)
-from verifiers.utils.interception_utils import (
-    create_empty_completion,
-    serialize_intercept_response,
-)
-
-
-def test_serialize_intercept_response_from_vf_response_uses_chat_completion_shape():
-    response = Response(
-        id="resp_1",
-        created=123,
-        model="test-model",
-        usage=Usage(
-            prompt_tokens=10,
-            reasoning_tokens=0,
-            completion_tokens=5,
-            total_tokens=15,
-        ),
-        message=ResponseMessage(
-            content=[TextContentPart(text="hello "), {"type": "text", "text": "world"}],
-            reasoning_content=None,
-            tool_calls=[
-                ToolCall(id="call_1", name="echo", arguments='{"x": 1}'),
-            ],
-            finish_reason="tool_calls",
-            is_truncated=False,
-            tokens=None,
-        ),
-    )
-
-    payload = serialize_intercept_response(response)
-
-    assert payload["id"] == "resp_1"
-    assert payload["object"] == "chat.completion"
-    assert payload["model"] == "test-model"
-    assert payload["choices"][0]["message"]["role"] == "assistant"
-    assert payload["choices"][0]["message"]["content"] == "hello world"
-    assert payload["choices"][0]["message"]["tool_calls"] == [
-        {
-            "id": "call_1",
-            "type": "function",
-            "function": {"name": "echo", "arguments": '{"x": 1}'},
-        }
-    ]
-    assert payload["choices"][0]["finish_reason"] == "tool_calls"
-    assert payload["usage"]["prompt_tokens"] == 10
-    assert payload["usage"]["completion_tokens"] == 5
-    assert payload["usage"]["total_tokens"] == 15
-
-
-def test_serialize_intercept_response_passthrough_native_chat_completion():
-    native = create_empty_completion("native-model")
-    payload = serialize_intercept_response(native)
-
-    assert payload["object"] == "chat.completion"
-    assert payload["model"] == "native-model"
-    assert len(payload["choices"]) == 1