verifiers 0.1.15.dev186__tar.gz → 0.1.15.dev188__tar.gz

{verifiers-0.1.15.dev186 → verifiers-0.1.15.dev188}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.1.15.dev186
+Version: 0.1.15.dev188
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.1.15.dev186 → verifiers-0.1.15.dev188}/tests/test_client_multimodal_types.py

@@ -3,7 +3,6 @@ from types import SimpleNamespace
 import pytest
 
 from verifiers.clients.openai_chat_completions_client import OpenAIChatCompletionsClient
-from verifiers.errors import EmptyModelResponseError
 from verifiers.types import (
     AssistantMessage,
     ImageUrlContentPart,
@@ -59,25 +58,6 @@ async def test_openai_to_native_prompt_with_typed_multimodal_content_parts():
     ]
 
 
-@pytest.mark.asyncio
-async def test_openai_chat_rejects_reasoning_only_native_response():
-    client = OpenAIChatCompletionsClient(object())
-    native_response = SimpleNamespace(
-        choices=[
-            SimpleNamespace(
-                message=_OpenAIMessage(
-                    content=None,
-                    reasoning_content="hidden chain",
-                    tool_calls=None,
-                )
-            )
-        ]
-    )
-
-    with pytest.raises(EmptyModelResponseError, match="reasoning but no content"):
-        await client.raise_from_native_response(native_response)
-
-
 @pytest.mark.asyncio
 async def test_openai_chat_accepts_refusal_with_reasoning_native_response():
     client = OpenAIChatCompletionsClient(object())
@@ -282,24 +262,6 @@ async def test_anthropic_from_native_response_always_parses_reasoning():
     assert response.message.content == "final answer"
 
 
-@pytest.mark.asyncio
-async def test_anthropic_rejects_reasoning_only_native_response():
-    pytest.importorskip("anthropic")
-    from verifiers.clients.anthropic_messages_client import AnthropicMessagesClient
-
-    client = AnthropicMessagesClient(object())
-    native_response = SimpleNamespace(
-        id="msg_think",
-        model="claude-haiku-4-5",
-        stop_reason="end_turn",
-        usage=SimpleNamespace(input_tokens=1, output_tokens=1),
-        content=[SimpleNamespace(type="thinking", thinking="hidden chain")],
-    )
-
-    with pytest.raises(EmptyModelResponseError, match="reasoning but no content"):
-        await client.raise_from_native_response(native_response)
-
-
 @pytest.mark.asyncio
 async def test_anthropic_tool_call_round_trips_thinking_blocks():
     pytest.importorskip("anthropic")

{verifiers-0.1.15.dev186 → verifiers-0.1.15.dev188}/tests/test_openai_responses_client.py

@@ -8,7 +8,6 @@ from verifiers.clients.openai_responses_client import (
     OPENAI_RESPONSES_OUTPUT_FIELD,
     OpenAIResponsesClient,
 )
-from verifiers.errors import EmptyModelResponseError
 from verifiers.types import (
     AssistantMessage,
     ClientConfig,
@@ -134,24 +133,6 @@ async def test_get_native_response_normalizes_sampling_args_and_tools():
     ]
 
 
-@pytest.mark.asyncio
-async def test_raise_from_native_response_rejects_reasoning_only_response():
-    native_response = SimpleNamespace(
-        output=[
-            {
-                "type": "reasoning",
-                "id": "rs_1",
-                "summary": [{"type": "summary_text", "text": "thinking"}],
-                "status": "completed",
-            }
-        ]
-    )
-    client = OpenAIResponsesClient(object())
-
-    with pytest.raises(EmptyModelResponseError, match="reasoning but no content"):
-        await client.raise_from_native_response(native_response)
-
-
 @pytest.mark.asyncio
 async def test_to_native_tool_omits_strict_when_unset():
     client = OpenAIResponsesClient(object())

{verifiers-0.1.15.dev186 → verifiers-0.1.15.dev188}/tests/test_renderer_client.py

@@ -355,14 +355,6 @@ async def test_renderer_client_rejects_empty_dict_native_response():
         await client.raise_from_native_response({})
 
 
-@pytest.mark.asyncio
-async def test_renderer_client_rejects_reasoning_only_native_response():
-    client = object.__new__(RendererClient)
-
-    with pytest.raises(EmptyModelResponseError, match="reasoning but no content"):
-        await client.raise_from_native_response({"reasoning_content": "hidden chain"})
-
-
 @pytest.mark.asyncio
 async def test_from_native_response_uses_request_id_and_token_lengths():
     """vLLM's /inference/v1/generate returns ``request_id`` (not ``id``) and

{verifiers-0.1.15.dev186 → verifiers-0.1.15.dev188}/verifiers/clients/anthropic_messages_client.py

@@ -390,14 +390,14 @@ class AnthropicMessagesClient(
                 has_text = True
             elif block_type == "tool_use":
                 has_tool_call = True
-            elif block_type in {"thinking", "redacted_thinking"}:
+            elif block_type == "thinking" and getattr(content_block, "thinking", None):
+                has_reasoning = True
+            elif block_type == "redacted_thinking" and getattr(
+                content_block, "data", None
+            ):
                 has_reasoning = True
 
-        if not (has_text or has_tool_call):
-            if has_reasoning:
-                raise EmptyModelResponseError(
-                    "Model returned reasoning but no content and did not call any tools"
-                )
+        if not (has_text or has_tool_call or has_reasoning):
             raise EmptyModelResponseError(
                 "Model returned no content and did not call any tools"
             )

{verifiers-0.1.15.dev186 → verifiers-0.1.15.dev188}/verifiers/clients/openai_chat_completions_client.py

@@ -100,6 +100,12 @@ def get_usage_field(usage: Any, key: str) -> Any:
     return getattr(usage, key, None)
 
 
+def _get_reasoning_tokens(usage: Any) -> int:
+    completion_details = get_usage_field(usage, "completion_tokens_details")
+    reasoning_tokens = get_usage_field(completion_details, "reasoning_tokens")
+    return reasoning_tokens if isinstance(reasoning_tokens, int) else 0
+
+
 def content_to_text(content: Any) -> str:
     """Get all text content from OAI message content."""
     if isinstance(content, str):
@@ -142,8 +148,19 @@ def parse_reasoning_content(message: Any) -> str | None:
 
     for field in DEFAULT_REASONING_FIELDS:
         value = message_dict.get(field)
-        if isinstance(value, str):
+        if isinstance(value, str) and value:
             return value
+    details = message_dict.get("reasoning_details")
+    if isinstance(details, list) and any(
+        isinstance(detail, Mapping)
+        and any(
+            detail.get(field)
+            for field in ("text", "summary", "data", "signature", "encrypted_content")
+        )
+        for detail in details
+    ):
+        # Keep structured provider reasoning as a replayable marker without flattening it.
+        return ""
     return None
 
 
@@ -345,12 +362,11 @@ class OpenAIChatCompletionsClient(
             or parse_refusal_content(message)
         )
         has_tool_calls = bool(getattr(message, "tool_calls", None))
-        has_reasoning = bool(parse_reasoning_content(message))
-        if not (has_content or has_tool_calls):
-            if has_reasoning:
-                raise EmptyModelResponseError(
-                    "Model returned reasoning but no content and did not call any tools"
-                )
+        has_reasoning = (
+            parse_reasoning_content(message) is not None
+            or _get_reasoning_tokens(getattr(response, "usage", None)) > 0
+        )
+        if not (has_content or has_tool_calls or has_reasoning):
             raise EmptyModelResponseError(
                 "Model returned no content and did not call any tools"
             )
@@ -442,6 +458,7 @@ class OpenAIChatCompletionsClient(
                 prompt_tokens = get_usage_field(usage, "input_tokens")
                 completion_tokens = get_usage_field(usage, "output_tokens")
             total_tokens = get_usage_field(usage, "total_tokens")
+            reasoning_tokens = _get_reasoning_tokens(usage)
             if not isinstance(prompt_tokens, int) or not isinstance(
                 completion_tokens, int
             ):
@@ -450,7 +467,7 @@ class OpenAIChatCompletionsClient(
                 total_tokens = prompt_tokens + completion_tokens
             return Usage(
                 prompt_tokens=prompt_tokens,
-                reasoning_tokens=0,
+                reasoning_tokens=reasoning_tokens,
                 completion_tokens=completion_tokens,
                 total_tokens=total_tokens,
             )
@@ -528,17 +545,29 @@ class OpenAIChatCompletionsClient(
         if not isinstance(model, str):
             model = ""
 
+        message = response.choices[0].message
+        content = parse_content(response)
+        reasoning_content = parse_reasoning_content(message)
+        tool_calls = parse_tool_calls(response)
+        if (
+            content is None
+            and reasoning_content is None
+            and not tool_calls
+            and _get_reasoning_tokens(getattr(response, "usage", None)) > 0
+        ):
+            content = ""
+
         return Response(
             id=response_id,
             created=created,
             model=model,
             usage=parse_usage(response),
             message=ResponseMessage(
-                content=parse_content(response),
-                reasoning_content=parse_reasoning_content(response.choices[0].message),
+                content=content,
+                reasoning_content=reasoning_content,
                 finish_reason=parse_finish_reason(response),
                 is_truncated=parse_is_truncated(response),
                 tokens=parse_tokens(response),
-                tool_calls=parse_tool_calls(response) or None,
+                tool_calls=tool_calls or None,
             ),
         )

{verifiers-0.1.15.dev186 → verifiers-0.1.15.dev188}/verifiers/clients/openai_responses_client.py

@@ -53,6 +53,12 @@ def _model_dump(value: Any) -> dict[str, Any]:
     return {}
 
 
+def _get_reasoning_tokens(usage: Any) -> int:
+    output_details = get_usage_field(usage, "output_tokens_details")
+    reasoning_tokens = get_usage_field(output_details, "reasoning_tokens")
+    return reasoning_tokens if isinstance(reasoning_tokens, int) else 0
+
+
 class OpenAIResponsesClient(
     Client[
         AsyncOpenAI,
@@ -171,7 +177,7 @@ class OpenAIResponsesClient(
                     return raw_items
 
                 items: list[dict[str, Any]] = []
-                if content_to_text(message.content):
+                if content_to_text(message.content) or message.content == "":
                     items.append(
                         {
                             "type": "message",
@@ -267,9 +273,14 @@ class OpenAIResponsesClient(
             message = _get_field(error, "message", "Model response failed")
             raise InvalidModelResponseError(str(message))
 
+        has_usage_reasoning = (
+            _get_reasoning_tokens(getattr(response, "usage", None)) > 0
+        )
         output = getattr(response, "output", None)
-        if output is None:
+        if output is None and not has_usage_reasoning:
             raise EmptyModelResponseError("Model returned no output")
+        if output is None:
+            output = []
         if not isinstance(output, Iterable):
             raise InvalidModelResponseError("Model returned invalid output")
 
@@ -293,11 +304,8 @@ class OpenAIResponsesClient(
                     ):
                         has_text = True
 
-        if not (has_text or has_tool_call):
-            if has_reasoning:
-                raise EmptyModelResponseError(
-                    "Model returned reasoning but no content and did not call any tools"
-                )
+        has_reasoning = has_reasoning or has_usage_reasoning
+        if not (has_text or has_tool_call or has_reasoning):
             raise EmptyModelResponseError(
                 "Model returned no content and did not call any tools"
             )
@@ -374,20 +382,13 @@ class OpenAIResponsesClient(
             prompt_tokens = get_usage_field(usage, "input_tokens")
             completion_tokens = get_usage_field(usage, "output_tokens")
             total_tokens = get_usage_field(usage, "total_tokens")
-            output_details = get_usage_field(usage, "output_tokens_details")
-            reasoning_tokens = (
-                get_usage_field(output_details, "reasoning_tokens")
-                if output_details is not None
-                else 0
-            )
+            reasoning_tokens = _get_reasoning_tokens(usage)
             if not isinstance(prompt_tokens, int) or not isinstance(
                 completion_tokens, int
             ):
                 return None
             if not isinstance(total_tokens, int):
                 total_tokens = prompt_tokens + completion_tokens
-            if not isinstance(reasoning_tokens, int):
-                reasoning_tokens = 0
             return Usage(
                 prompt_tokens=prompt_tokens,
                 reasoning_tokens=reasoning_tokens,
@@ -421,14 +422,27 @@ class OpenAIResponsesClient(
         if not isinstance(model, str):
             model = ""
 
+        output_items = raw_output_items(response)
+        content = parse_content(response)
+        reasoning_content = parse_reasoning_content(response)
+        tool_calls = parse_tool_calls(response)
+        if (
+            not output_items
+            and content is None
+            and reasoning_content is None
+            and not tool_calls
+            and _get_reasoning_tokens(getattr(response, "usage", None)) > 0
+        ):
+            content = ""
+
         message_data: dict[str, Any] = {
-            "content": parse_content(response),
-            "reasoning_content": parse_reasoning_content(response),
+            "content": content,
+            "reasoning_content": reasoning_content,
             "finish_reason": parse_finish_reason(response),
             "is_truncated": parse_is_truncated(response),
             "tokens": None,
-            "tool_calls": parse_tool_calls(response) or None,
-            OPENAI_RESPONSES_OUTPUT_FIELD: raw_output_items(response),
+            "tool_calls": tool_calls or None,
+            OPENAI_RESPONSES_OUTPUT_FIELD: output_items,
         }
 
         return Response(

{verifiers-0.1.15.dev186 → verifiers-0.1.15.dev188}/verifiers/clients/renderer_client.py

@@ -641,11 +641,7 @@ class RendererClient(
         # model having tried to call a tool, so we don't filter by status here.
         has_tool_calls = bool(response.get("tool_calls"))
         has_reasoning = bool(response.get("reasoning_content"))
-        if not (has_content or has_tool_calls):
-            if has_reasoning:
-                raise EmptyModelResponseError(
-                    "Model returned reasoning but no content and did not call any tools"
-                )
+        if not (has_content or has_tool_calls or has_reasoning):
             raise EmptyModelResponseError(
                 "Model returned no content and did not call any tools"
             )

{verifiers-0.1.15.dev186 → verifiers-0.1.15.dev188}/verifiers/envs/experimental/composable/tasksets/__init__.py

@@ -32,6 +32,10 @@ from verifiers.envs.experimental.composable.tasksets.harbor.harbor import (
     HarborRubric,
     HarborTaskSet,
 )
+from verifiers.envs.experimental.composable.tasksets.harbor.cli_gym import (
+    CLIGymTaskSet,
+    make_cli_gym_taskset,
+)
 from verifiers.envs.experimental.composable.tasksets.harbor.terminal_lego import (
     TerminalLegoTaskSet,
     make_terminal_lego_taskset,
@@ -64,6 +68,8 @@ __all__ = [
     "HarborDatasetTaskSet",
     "HarborRubric",
     "HarborDatasetRubric",
+    "CLIGymTaskSet",
+    "make_cli_gym_taskset",
     "TerminalLegoTaskSet",
     "make_terminal_lego_taskset",
 ]

{verifiers-0.1.15.dev186 → verifiers-0.1.15.dev188}/verifiers/envs/experimental/composable/tasksets/harbor/__init__.py

@@ -4,6 +4,7 @@ from .harbor import (
     HarborRubric,
     HarborTaskSet,
 )
+from .cli_gym import CLIGymTaskSet, make_cli_gym_taskset
 from .terminal_lego import TerminalLegoTaskSet, make_terminal_lego_taskset
 
 __all__ = [
@@ -11,6 +12,8 @@ __all__ = [
     "HarborDatasetTaskSet",
     "HarborRubric",
     "HarborDatasetRubric",
+    "CLIGymTaskSet",
+    "make_cli_gym_taskset",
     "TerminalLegoTaskSet",
     "make_terminal_lego_taskset",
 ]