verifiers 0.1.15.dev167__tar.gz → 0.1.15.dev168__tar.gz

{verifiers-0.1.15.dev167 → verifiers-0.1.15.dev168}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.1.15.dev167
+Version: 0.1.15.dev168
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.1.15.dev167 → verifiers-0.1.15.dev168}/tests/test_environment.py

@@ -697,6 +697,30 @@ class TestMaybeRetry:
         error_data = rollout_outputs[0]["error"]
         assert "InfraError" == error_data["error"]
 
+    @pytest.mark.asyncio
+    async def test_retries_serialized_infra_error_subclass(self):
+        """A serialized InfraError subclass (e.g. SandboxError) in returned state
+        must trigger retry.
+
+        The v1 harness serializes state["error"] to ErrorData before maybe_retry
+        inspects it, so matching must be subclass-aware (rebuild concrete error +
+        isinstance) — base-name substring matching missed SandboxError, which is
+        an InfraError and should be retried.
+        """
+        from verifiers.utils.async_utils import maybe_retry
+        from verifiers.utils.error_utils import error_data
+
+        serialized = error_data(vf.SandboxError("Program file upload failed"))
+        calls = {"n": 0}
+
+        async def attempt():
+            calls["n"] += 1
+            return {"error": serialized}
+
+        result = await maybe_retry(attempt, max_retries=2, initial=0.0, max_wait=0.0)()
+        assert calls["n"] == 3  # 1 initial + 2 retries (InfraError is retryable)
+        assert result["error"] == serialized  # last result returned after exhaustion
+
 
 class TestEmptyModelResponseErrors:
     """Test cases for empty and invalid model response error handling."""

{verifiers-0.1.15.dev167 → verifiers-0.1.15.dev168}/tests/test_trajectory_processing.py

@@ -282,6 +282,49 @@ async def test_parsed_prompt_attribution_survives_v1_assert_serializable():
     State({"trajectory": [step]}).assert_serializable()
 
 
+def test_assert_serializable_accepts_msgpack_sidecars_rejects_unknown():
+    """The ``assert_serializable`` json.dumps gate must accept exactly what the
+    trainer transport (msgpack) accepts, while staying strict otherwise.
+
+    Trajectory token steps carry sidecars that are non-JSON by design and reach
+    the trainer via msgpack, not JSON: the renderer ``MultiModalData`` (a
+    dataclass holding numpy pixel arrays) and ``routed_experts`` (a raw
+    ``memoryview`` buffer). Both must clear the gate; any other
+    non-serializable object must still raise.
+    """
+    import dataclasses
+
+    import numpy as np
+
+    @dataclasses.dataclass
+    class _FakeMultiModalData:
+        mm_hashes: dict
+        mm_items: dict
+        mm_placeholders: dict
+
+    mm = _FakeMultiModalData(
+        mm_hashes={"image": ["h1"]},
+        mm_items={"image": [np.zeros((2, 2), dtype=np.uint8)]},
+        mm_placeholders={"image": [{"offset": 0, "length": 4}]},
+    )
+    step = {
+        "tokens": {
+            "prompt_ids": [1, 2],
+            "multi_modal_data": mm,
+            "routed_experts": {"data": memoryview(b"abc"), "shape": [3], "start": 0},
+        }
+    }
+    # Must not raise: both sidecars are msgpack-transported, not JSON.
+    State({"trajectory": [step]}).assert_serializable()
+
+    # A genuinely non-serializable object must still be rejected.
+    class _Unknown:
+        pass
+
+    with pytest.raises(TypeError):
+        State({"trajectory": [{"tokens": _Unknown()}]}).assert_serializable()
+
+
 def test_process_trajectory_steps_for_training(make_input):
     """Test processing trajectory steps into training examples."""
     state1 = State(

{verifiers-0.1.15.dev167 → verifiers-0.1.15.dev168}/tests/test_v1_runtime_lifecycle.py

@@ -110,6 +110,15 @@ class BlockingModelClient(CapturingModelClient):
         return await super().get_response(**kwargs)
 
 
+class RaisingModelClient:
+    def __init__(self, error: vf.Error):
+        self.error = error
+
+    async def get_response(self, **kwargs: object) -> Response:
+        _ = kwargs
+        raise self.error
+
+
 class FakeCreateSandboxRequest:
     def __init__(self, **kwargs: object):
         self.kwargs = kwargs
@@ -517,6 +526,31 @@ async def endpoint_program(task, state):
     }
 
 
+async def endpoint_model_error_program(task, state):
+    _ = task
+    root = state["endpoint_root_url"].rstrip("/")
+    endpoint_client = cast(OpenAI, state.get_client(api="chat", sync=True))
+    auth_headers = {"Authorization": f"Bearer {endpoint_client.api_key}"}
+    endpoint_client.close()
+
+    def post_model() -> None:
+        request = urllib.request.Request(
+            f"{root}/vf/model",
+            data=json.dumps(
+                {"messages": [{"role": "user", "content": "too long"}]}
+            ).encode(),
+            headers={"content-type": "application/json", **auth_headers},
+        )
+        with urllib.request.urlopen(request):
+            pass
+
+    try:
+        await asyncio.to_thread(post_model)
+    except Exception as exc:
+        raise vf.SandboxError("Sandbox command failed") from exc
+    raise AssertionError("Expected /vf/model to fail")
+
+
 async def endpoint_trajectory_program(task, state):
     _ = task
     root = state["endpoint_root_url"].rstrip("/")
@@ -725,6 +759,7 @@ for _name, _value in {
     "initialize_from_taskset": initialize_from_taskset,
     "child_reads_program_sandbox": child_reads_program_sandbox,
     "endpoint_program": endpoint_program,
+    "endpoint_model_error_program": endpoint_model_error_program,
     "endpoint_trajectory_program": endpoint_trajectory_program,
     "concurrent_endpoint_program": concurrent_endpoint_program,
     "mcp_proxy_program": mcp_proxy_program,
@@ -827,6 +862,41 @@ async def test_endpoint_exposes_tool_user_and_stop_surfaces() -> None:
     assert "endpoint_root_url" not in state
 
 
+@pytest.mark.asyncio
+async def test_vf_model_bridge_preserves_overlong_prompt_error() -> None:
+    harness = make_harness(
+        program={"fn": program_ref("endpoint_model_error_program")},
+        model="test-model",
+        client=RaisingModelClient(vf.OverlongPromptError("too long")),
+    )
+    task = vf.Task({"prompt": [{"role": "user", "content": "hi"}]}).freeze()
+
+    state = await harness.run(task)
+    await harness.teardown()
+
+    assert state["prompt_too_long"] is True
+    assert state["is_truncated"] is True
+    assert state["stop_condition"] == "prompt_too_long"
+    assert state.get("error") is None
+
+
+@pytest.mark.asyncio
+async def test_vf_model_bridge_preserves_model_error() -> None:
+    harness = make_harness(
+        program={"fn": program_ref("endpoint_model_error_program")},
+        model="test-model",
+        client=RaisingModelClient(vf.ModelError("model failed")),
+    )
+    task = vf.Task({"prompt": [{"role": "user", "content": "hi"}]}).freeze()
+
+    state = await harness.run(task)
+    await harness.teardown()
+
+    assert state["stop_condition"] == "has_error"
+    assert state["error"]["error"] == "ModelError"
+    assert "SandboxError" not in state["error"]["error_chain_str"]
+
+
 @pytest.mark.asyncio
 async def test_endpoint_request_can_hide_internal_model_call_from_trajectory() -> None:
     client = FakeModelClient([fake_response("hidden"), fake_response("shown")])
@@ -1462,6 +1532,70 @@ async def test_create_sandbox_cleans_up_wait_failure_with_retry(
     assert client.delete_calls == 2
 
 
+@pytest.mark.asyncio
+async def test_upload_program_files_retries_transient_transfer_error(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    install_fake_sandboxes(monkeypatch)
+    disable_sandbox_retry_sleep(monkeypatch)
+
+    class FlakyUploadClient:
+        calls = 0
+
+        async def upload_bytes(self, *args: object, **kwargs: object) -> None:
+            _ = args, kwargs
+            self.calls += 1
+            if self.calls == 1:
+                raise FakeAPIError("Upload failed: ")
+
+    client = FlakyUploadClient()
+    task = vf.Task({"prompt": [{"role": "user", "content": "hi"}]}).freeze()
+    state = vf.State.for_task(task)
+
+    await sandbox_utils.upload_program_files(
+        cast(sandbox_utils.SandboxClient, client),
+        "sbx-upload",
+        {"files": {"/tmp/file.txt": "content"}},
+        task,
+        state,
+        Runtime(),
+    )
+
+    assert client.calls == 2
+
+
+@pytest.mark.asyncio
+async def test_upload_program_files_does_not_retry_non_transient_api_error(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    install_fake_sandboxes(monkeypatch)
+    disable_sandbox_retry_sleep(monkeypatch)
+
+    class FailingUploadClient:
+        calls = 0
+
+        async def upload_bytes(self, *args: object, **kwargs: object) -> None:
+            _ = args, kwargs
+            self.calls += 1
+            raise FakeAPIError("Upload failed: HTTP 400: bad request")
+
+    client = FailingUploadClient()
+    task = vf.Task({"prompt": [{"role": "user", "content": "hi"}]}).freeze()
+    state = vf.State.for_task(task)
+
+    with pytest.raises(vf.SandboxError, match="HTTP 400"):
+        await sandbox_utils.upload_program_files(
+            cast(sandbox_utils.SandboxClient, client),
+            "sbx-upload",
+            {"files": {"/tmp/file.txt": "content"}},
+            task,
+            state,
+            Runtime(),
+        )
+
+    assert client.calls == 1
+
+
 @pytest.mark.asyncio
 async def test_create_sandbox_cancellation_deletes_late_provider_result(
     monkeypatch: pytest.MonkeyPatch,
@@ -1603,8 +1737,8 @@ async def test_sandbox_base_program_max_turns_zero_is_unbounded(
     config_path.write_text(json.dumps({"max_turns": 0}))
     namespace["RUNNER_CONFIG_PATH"] = str(config_path)
 
-    async def create_model_message(state, messages, client):
-        _ = state, messages, client
+    async def create_model_message(state, messages):
+        _ = state, messages
         return {"role": "assistant", "content": "done"}
 
     async def call_user(state, messages):
@@ -1621,12 +1755,55 @@ async def test_sandbox_base_program_max_turns_zero_is_unbounded(
 
     state = {"prompt": [{"role": "user", "content": "hi"}], "runtime": {}}
     run_base = cast(Any, namespace["run_base"])
-    result = await run_base({}, state, object())
+    result = await run_base({}, state)
 
     assert result["completion"] == [{"role": "assistant", "content": "done"}]
     assert result["stop_condition"] == "no_tools"
 
 
+@pytest.mark.asyncio
+async def test_sandbox_base_program_model_call_uses_vf_model_bridge() -> None:
+    namespace: dict[str, object] = {}
+    source = runner_source().rsplit("asyncio.run(main())", 1)[0]
+    exec(source, namespace)
+
+    posted: list[tuple[str, Any, object]] = []
+
+    async def vf_post(state, path, payload, timeout=None):
+        _ = state
+        posted.append((path, payload, timeout))
+        return {"message": {"role": "assistant", "content": "ok"}}
+
+    namespace["vf_post"] = vf_post
+    create_model_message = cast(Any, namespace["create_model_message"])
+
+    # Canonical Messages (incl. an image content part) are sent unchanged over the
+    # /vf/model bridge; the host owns client resolution + tokenization and returns
+    # the assistant message.
+    messages = [
+        {"role": "user", "content": "hi"},
+        {
+            "role": "tool",
+            "tool_call_id": "call_1",
+            "content": [
+                {"type": "text", "text": "shot"},
+                {
+                    "type": "image_url",
+                    "image_url": {"url": "data:image/png;base64,AAA"},
+                },
+            ],
+        },
+    ]
+    message = await create_model_message({"runtime": {}}, messages)
+
+    assert message == {"role": "assistant", "content": "ok"}
+    assert len(posted) == 1
+    path, payload, timeout = posted[0]
+    assert path == "model"
+    assert payload["messages"] == messages  # image part preserved verbatim
+    assert timeout is None
+
+
 def test_sandbox_program_patch_cannot_set_lifecycle_fields() -> None:
     state = vf.State.for_task(vf.Task({"prompt": []}).freeze())
 

{verifiers-0.1.15.dev167 → verifiers-0.1.15.dev168}/verifiers/clients/anthropic_messages_client.py

@@ -214,15 +214,15 @@ class AnthropicMessagesClient(
             return {}
 
         def build_tool_result_block(message: ToolMessage) -> ToolResultBlockParam:
+            if isinstance(message.content, str):
+                result_content: Any = message.content
+            else:
+                # Keep images: image_url parts -> Anthropic image blocks (not "[image]" text).
+                result_content = normalize_anthropic_content(message.content)
             return ToolResultBlockParam(
                 type="tool_result",
                 tool_use_id=message.tool_call_id,
-                content=cast(
-                    Any,
-                    message.content
-                    if isinstance(message.content, str)
-                    else " ".join(content_to_text_chunks(message.content)),
-                ),
+                content=cast(Any, result_content),
             )
 
         def from_chat_message(message: Message) -> AnthropicMessageParam | None:

{verifiers-0.1.15.dev167 → verifiers-0.1.15.dev168}/verifiers/clients/openai_responses_client.py

@@ -156,8 +156,8 @@ class OpenAIResponsesClient(
             if isinstance(message, ToolMessage):
                 output = message.content
                 if not isinstance(output, str):
-                    text = content_to_text(output)
-                    output = text if text else str(output)
+                    # Keep images: image_url parts -> Responses input_image (not text).
+                    output = normalize_message_content(output)
                 return [
                     {
                         "type": "function_call_output",

{verifiers-0.1.15.dev167 → verifiers-0.1.15.dev168}/verifiers/types.py

@@ -1081,9 +1081,43 @@ def _strip_runtime_handles(value: object) -> None:
             _strip_runtime_handles(item)
 
 
+def _json_gate_default(obj: object) -> object:
+    # Gate parity with the trainer transport (serve_utils.msgpack_encoder):
+    # accept what msgpack ships (it reaches the trainer via msgpack, not JSON),
+    # as cheap stand-ins; json re-invokes this on nested values. Unknown raises.
+    import dataclasses
+    from datetime import date, datetime
+    from enum import Enum
+
+    if isinstance(obj, (Path, uuid.UUID)):
+        return str(obj)  # filesystem path / uuid
+    if isinstance(obj, Enum):
+        return obj.value  # enum member
+    if isinstance(obj, (datetime, date)):
+        return obj.isoformat()  # timestamps
+    if isinstance(obj, (bytes, bytearray, memoryview)):
+        return f"<{type(obj).__name__}>"  # raw buffers (e.g. routed_experts data)
+    np = sys.modules.get("numpy")
+    if np is not None and isinstance(obj, (np.integer, np.floating)):
+        return obj.item()  # numpy scalar -> python scalar
+    if np is not None and isinstance(obj, np.ndarray):
+        return {"dtype": str(obj.dtype), "shape": list(obj.shape)}  # array: shape only
+    torch = sys.modules.get("torch")
+    if torch is not None and isinstance(obj, torch.Tensor):
+        return {"dtype": str(obj.dtype), "shape": list(obj.shape)}  # tensor: shape only
+    model_dump = getattr(obj, "model_dump", None)
+    if callable(model_dump):
+        return model_dump(exclude_none=True)  # pydantic model
+    if dataclasses.is_dataclass(obj) and not isinstance(obj, type):
+        # renderer sidecars (MultiModalData / PlaceholderRange); shallow so json
+        # recurses into fields without deep-copying pixel arrays.
+        return {f.name: getattr(obj, f.name) for f in dataclasses.fields(obj)}
+    raise TypeError(f"{type(obj).__name__} is not JSON-serializable")
+
+
 def assert_json_serializable(value: object) -> None:
     try:
-        json.dumps(value)
+        json.dumps(value, default=_json_gate_default)
     except TypeError as e:
         raise TypeError("Task and State values must be JSON-serializable.") from e
 

{verifiers-0.1.15.dev167 → verifiers-0.1.15.dev168}/verifiers/utils/async_utils.py

@@ -13,7 +13,7 @@ from pydantic import BaseModel
 
 import verifiers as vf
 from verifiers.utils.error_utils import ErrorChain
-from verifiers.utils.error_utils import error_data_to_exception
+from verifiers.utils.error_utils import error_from_data, is_error_data
 from verifiers.utils.logging_utils import print_time
 
 logger = logging.getLogger(__name__)
@@ -161,25 +161,24 @@ def maybe_retry(
     if max_retries <= 0:
         return func
 
+    def reraise_one(err, error_types: tuple[type[Exception], ...]):
+        if not err:
+            return
+        if any(isinstance(err, err_type) for err_type in error_types):
+            raise err
+        # Rebuild serialized ErrorData so base types match (SandboxError -> InfraError).
+        if isinstance(err, Mapping) and is_error_data(err):
+            rebuilt = error_from_data(err)
+            if isinstance(rebuilt, error_types):
+                raise rebuilt
+
     def reraise_error_from_state(result, error_types: tuple[type[Exception], ...]):
         """Re-raise specified errors from state(s) to trigger tenacity retry."""
         if isinstance(result, dict):
-            err = result.get("error")
-            if err and any(isinstance(err, err_type) for err_type in error_types):
-                raise err
-            if isinstance(err, Mapping):
-                retry_error = error_data_to_exception(err, error_types)
-                if retry_error is not None:
-                    raise retry_error
+            reraise_one(result.get("error"), error_types)
         elif isinstance(result, list):
             for state in result:
-                err = state.get("error")
-                if err and any(isinstance(err, err_type) for err_type in error_types):
-                    raise err
-                if isinstance(err, Mapping):
-                    retry_error = error_data_to_exception(err, error_types)
-                    if retry_error is not None:
-                        raise retry_error
+                reraise_one(state.get("error"), error_types)
 
     def log_retry(retry_state: tc.RetryCallState) -> None:
         """Log a warning with the exception and the number of attempts."""

{verifiers-0.1.15.dev167 → verifiers-0.1.15.dev168}/verifiers/utils/interception_utils.py

@@ -120,6 +120,10 @@ class InterceptionServer:
                 "/rollout/{rollout_id}/vf/stop",
                 self._handle_stop_request,
             )
+            app.router.add_post(
+                "/rollout/{rollout_id}/vf/model",
+                self._handle_model_request,
+            )
             app.router.add_get(
                 "/health",
                 lambda _: web.json_response({"status": "ok"}),
@@ -192,6 +196,7 @@ class InterceptionServer:
         tool_defs: Any | None = None,
         user_handler: Any | None = None,
         stop_handler: Any | None = None,
+        model_handler: Any | None = None,
     ) -> asyncio.Queue:
         request_queue: asyncio.Queue = asyncio.Queue()
         self.active_rollouts[rollout_id] = {
@@ -201,6 +206,7 @@ class InterceptionServer:
             "tool_defs": tool_defs,
             "user_handler": user_handler,
             "stop_handler": stop_handler,
+            "model_handler": model_handler,
         }
         return request_queue
 
@@ -415,6 +421,38 @@ class InterceptionServer:
             return web.json_response({"error": str(e)}, status=500)
         return web.json_response(result)
 
+    async def _handle_model_request(self, request: Any) -> Any:
+        if not self._authorized(request):
+            return web.json_response({"error": "Unauthorized"}, status=401)
+
+        rollout_id = request.match_info["rollout_id"]
+        context = self.active_rollouts.get(rollout_id)
+        if not context:
+            return web.json_response({"error": "Rollout not found"}, status=404)
+        model_handler = context.get("model_handler")
+        if model_handler is None:
+            return web.json_response({"error": "Model proxy unavailable"}, status=404)
+
+        try:
+            request_body = await request.json()
+        except Exception as e:
+            return web.json_response({"error": f"Invalid JSON: {e}"}, status=400)
+        messages = request_body.get("messages")
+        if not isinstance(messages, list):
+            return web.json_response(
+                {"error": "Model request messages must be a list"}, status=400
+            )
+        tools = request_body.get("tools")
+
+        try:
+            result = model_handler(messages, tools)
+            if inspect.isawaitable(result):
+                result = await result
+            message = jsonable(result)
+        except Exception as e:
+            return web.json_response({"error": str(e)}, status=500)
+        return web.json_response({"message": message})
+
     async def _handle_streaming_response(
         self, http_request: Any, rollout_id: str, intercept: dict
     ) -> Any:

{verifiers-0.1.15.dev167 → verifiers-0.1.15.dev168}/verifiers/v1/harness.py

@@ -6,7 +6,7 @@ from pydantic import AliasChoices, Field
 
 import verifiers as vf
 from verifiers.clients.client import Client
-from verifiers.errors import Error, OverlongPromptError
+from verifiers.errors import Error, OverlongPromptError, SandboxError
 from verifiers.types import (
     ClientConfig,
     MessageContent,
@@ -93,7 +93,7 @@ from .types import (
 if TYPE_CHECKING:
     from .taskset import Taskset
 
-ProgramResult: TypeAlias = State | ConfigData | None
+ProgramResult: TypeAlias = State | JsonData | None
 ProgramRunner: TypeAlias = Callable[[Task, State], Awaitable[ProgramResult]]
 
 
@@ -282,6 +282,13 @@ class Harness(RuntimeOwnerMixin[ConfigT], Generic[ConfigT]):
         return state
 
     def record_error(self, state: State, error: Error) -> None:
+        if state.get("prompt_too_long"):
+            state._set_truncated(True)
+            state._set_stop_condition("prompt_too_long", overwrite=True)
+            return
+        if isinstance(error, SandboxError) and isinstance(state.get("error"), Error):
+            state._set_stop_condition("has_error", overwrite=True)
+            return
         if isinstance(error, OverlongPromptError):
             state["prompt_too_long"] = True
             state._set_truncated(True)