kiln-ai 0.14.0__py3-none-any.whl → 0.16.0__py3-none-any.whl

kiln_ai/adapters/model_adapters/base_adapter.py

@@ -3,9 +3,12 @@ from abc import ABCMeta, abstractmethod
 from dataclasses import dataclass
 from typing import Dict, Literal, Tuple
 
+import jsonschema
+
 from kiln_ai.adapters.ml_model_list import KilnModelProvider, StructuredOutputMode
 from kiln_ai.adapters.parsers.json_parser import parse_json_string
 from kiln_ai.adapters.parsers.parser_registry import model_parser_from_id
+from kiln_ai.adapters.parsers.request_formatters import request_formatter_from_id
 from kiln_ai.adapters.prompt_builders import prompt_builder_from_id
 from kiln_ai.adapters.provider_tools import kiln_model_provider_from
 from kiln_ai.adapters.run_output import RunOutput
@@ -15,8 +18,9 @@ from kiln_ai.datamodel import (
     Task,
     TaskOutput,
     TaskRun,
+    Usage,
 )
-from kiln_ai.datamodel.json_schema import validate_schema
+from kiln_ai.datamodel.json_schema import validate_schema_with_value_error
 from kiln_ai.datamodel.task import RunConfig
 from kiln_ai.utils.config import Config
 
@@ -103,16 +107,26 @@ class BaseAdapter(metaclass=ABCMeta):
         if self.input_schema is not None:
             if not isinstance(input, dict):
                 raise ValueError(f"structured input is not a dict: {input}")
-            validate_schema(input, self.input_schema)
+
+            validate_schema_with_value_error(
+                input,
+                self.input_schema,
+                "This task requires a specific input schema. While the model produced JSON, that JSON didn't meet the schema. Search 'Troubleshooting Structured Data Issues' in our docs for more information.",
+            )
+
+        # Format model input for model call (we save the original input in the task without formatting)
+        formatted_input = input
+        formatter_id = self.model_provider().formatter
+        if formatter_id is not None:
+            formatter = request_formatter_from_id(formatter_id)
+            formatted_input = formatter.format_input(input)
 
         # Run
-        run_output = await self._run(input)
+        run_output, usage = await self._run(formatted_input)
 
         # Parse
         provider = self.model_provider()
-        parser = model_parser_from_id(provider.parser)(
-            structured_output=self.has_structured_output()
-        )
+        parser = model_parser_from_id(provider.parser)
         parsed_output = parser.parse_output(original_output=run_output)
 
         # validate output
@@ -125,7 +139,11 @@ class BaseAdapter(metaclass=ABCMeta):
                 raise RuntimeError(
                     f"structured response is not a dict: {parsed_output.output}"
                 )
-            validate_schema(parsed_output.output, self.output_schema)
+            validate_schema_with_value_error(
+                parsed_output.output,
+                self.output_schema,
+                "This task requires a specific output schema. While the model produced JSON, that JSON didn't meet the schema. Search 'Troubleshooting Structured Data Issues' in our docs for more information.",
+            )
         else:
             if not isinstance(parsed_output.output, str):
                 raise RuntimeError(
@@ -142,7 +160,7 @@ class BaseAdapter(metaclass=ABCMeta):
             )
 
         # Generate the run and output
-        run = self.generate_run(input, input_source, parsed_output)
+        run = self.generate_run(input, input_source, parsed_output, usage)
 
         # Save the run if configured to do so, and we have a path to save to
         if (
@@ -165,7 +183,7 @@ class BaseAdapter(metaclass=ABCMeta):
         pass
 
     @abstractmethod
-    async def _run(self, input: Dict | str) -> RunOutput:
+    async def _run(self, input: Dict | str) -> Tuple[RunOutput, Usage | None]:
         pass
 
     def build_prompt(self) -> str:
@@ -204,7 +222,11 @@ class BaseAdapter(metaclass=ABCMeta):
 
     # create a run and task output
     def generate_run(
-        self, input: Dict | str, input_source: DataSource | None, run_output: RunOutput
+        self,
+        input: Dict | str,
+        input_source: DataSource | None,
+        run_output: RunOutput,
+        usage: Usage | None = None,
     ) -> TaskRun:
         # Convert input and output to JSON strings if they are dictionaries
         input_str = (
@@ -237,6 +259,7 @@ class BaseAdapter(metaclass=ABCMeta):
             ),
             intermediate_outputs=run_output.intermediate_outputs,
             tags=self.base_adapter_config.default_tags or [],
+            usage=usage,
         )
 
         return new_task_run

kiln_ai/adapters/model_adapters/litellm_adapter.py

@@ -1,7 +1,9 @@
+import logging
 from typing import Any, Dict
 
 import litellm
 from litellm.types.utils import ChoiceLogprobs, Choices, ModelResponse
+from litellm.types.utils import Usage as LiteLlmUsage
 
 import kiln_ai.datamodel as datamodel
 from kiln_ai.adapters.ml_model_list import (
@@ -14,14 +16,15 @@ from kiln_ai.adapters.model_adapters.base_adapter import (
     AdapterConfig,
     BaseAdapter,
     RunOutput,
+    Usage,
 )
-from kiln_ai.adapters.model_adapters.litellm_config import (
-    LiteLlmConfig,
-)
+from kiln_ai.adapters.model_adapters.litellm_config import LiteLlmConfig
 from kiln_ai.datamodel import PromptGenerators, PromptId
 from kiln_ai.datamodel.task import RunConfig
 from kiln_ai.utils.exhaustive_error import raise_exhaustive_enum_error
 
+logger = logging.getLogger(__name__)
+
 
 class LiteLlmAdapter(BaseAdapter):
     def __init__(
@@ -49,7 +52,7 @@ class LiteLlmAdapter(BaseAdapter):
             config=base_adapter_config,
         )
 
-    async def _run(self, input: Dict | str) -> RunOutput:
+    async def _run(self, input: Dict | str) -> tuple[RunOutput, Usage | None]:
         provider = self.model_provider()
         if not provider.model_id:
             raise ValueError("Model ID is required for OpenAI compatible models")
@@ -65,6 +68,7 @@ class LiteLlmAdapter(BaseAdapter):
         run_strategy, cot_prompt = self.run_strategy()
 
         if run_strategy == "cot_as_message":
+            # Used for reasoning-capable models that can output thinking and structured format
             if not cot_prompt:
                 raise ValueError("cot_prompt is required for cot_as_message strategy")
             messages.append({"role": "system", "content": cot_prompt})
@@ -73,9 +77,11 @@ class LiteLlmAdapter(BaseAdapter):
                 raise ValueError("cot_prompt is required for cot_two_call strategy")
             messages.append({"role": "system", "content": cot_prompt})
 
-            # First call for chain of thought - No logprobs as only needed for final answer
+            # First call for chain of thought
+            # No response format as this request is for "thinking" in plain text
+            # No logprobs as only needed for final answer
             completion_kwargs = await self.build_completion_kwargs(
-                provider, messages, None
+                provider, messages, None, skip_response_format=True
             )
             cot_response = await litellm.acompletion(**completion_kwargs)
             if (
@@ -136,8 +142,12 @@ class LiteLlmAdapter(BaseAdapter):
             raise RuntimeError("Logprobs were required, but no logprobs were returned.")
 
         # Save reasoning if it exists and was parsed by LiteLLM (or openrouter, or anyone upstream)
-        if hasattr(message, "reasoning_content") and message.reasoning_content:
-            intermediate_outputs["reasoning"] = message.reasoning_content
+        if (
+            hasattr(message, "reasoning_content")
+            and message.reasoning_content
+            and len(message.reasoning_content.strip()) > 0
+        ):
+            intermediate_outputs["reasoning"] = message.reasoning_content.strip()
 
         # the string content of the response
         response_content = message.content
@@ -166,7 +176,7 @@ class LiteLlmAdapter(BaseAdapter):
             output=response_content,
             intermediate_outputs=intermediate_outputs,
             output_logprobs=logprobs,
-        )
+        ), self.usage_from_response(response)
 
     def adapter_name(self) -> str:
         return "kiln_openai_compatible_adapter"
@@ -367,6 +377,7 @@ class LiteLlmAdapter(BaseAdapter):
         provider: KilnModelProvider,
         messages: list[dict[str, Any]],
         top_logprobs: int | None,
+        skip_response_format: bool = False,
     ) -> dict[str, Any]:
         extra_body = self.build_extra_body(provider)
 
@@ -380,12 +391,40 @@ class LiteLlmAdapter(BaseAdapter):
             **self._additional_body_options,
         }
 
-        # Response format: json_schema, json_instructions, json_mode, function_calling, etc
-        response_format_options = await self.response_format_options()
-        completion_kwargs.update(response_format_options)
+        if not skip_response_format:
+            # Response format: json_schema, json_instructions, json_mode, function_calling, etc
+            response_format_options = await self.response_format_options()
+            completion_kwargs.update(response_format_options)
 
         if top_logprobs is not None:
             completion_kwargs["logprobs"] = True
             completion_kwargs["top_logprobs"] = top_logprobs
 
         return completion_kwargs
+
+    def usage_from_response(self, response: ModelResponse) -> Usage | None:
+        litellm_usage = response.get("usage", None)
+        cost = response._hidden_params.get("response_cost", None)
+        if not litellm_usage and not cost:
+            return None
+
+        usage = Usage()
+
+        if litellm_usage and isinstance(litellm_usage, LiteLlmUsage):
+            usage.input_tokens = litellm_usage.get("prompt_tokens", None)
+            usage.output_tokens = litellm_usage.get("completion_tokens", None)
+            usage.total_tokens = litellm_usage.get("total_tokens", None)
+        else:
+            logger.warning(
+                f"Unexpected usage format from litellm: {litellm_usage}. Expected Usage object, got {type(litellm_usage)}"
+            )
+
+        if isinstance(cost, float):
+            usage.cost = cost
+        elif cost is not None:
+            # None is allowed, but no other types are expected
+            logger.warning(
+                f"Unexpected cost format from litellm: {cost}. Expected float, got {type(cost)}"
+            )
+
+        return usage

kiln_ai/adapters/model_adapters/test_base_adapter.py

@@ -3,7 +3,8 @@ from unittest.mock import MagicMock, patch
 import pytest
 
 from kiln_ai.adapters.ml_model_list import KilnModelProvider, StructuredOutputMode
-from kiln_ai.adapters.model_adapters.base_adapter import BaseAdapter
+from kiln_ai.adapters.model_adapters.base_adapter import BaseAdapter, RunOutput
+from kiln_ai.adapters.parsers.request_formatters import request_formatter_from_id
 from kiln_ai.datamodel import Task
 from kiln_ai.datamodel.task import RunConfig
 
@@ -12,7 +13,7 @@ class MockAdapter(BaseAdapter):
     """Concrete implementation of BaseAdapter for testing"""
 
     async def _run(self, input):
-        return None
+        return None, None
 
     def adapter_name(self) -> str:
         return "test"
@@ -42,6 +43,22 @@ def adapter(base_task):
     )
 
 
+@pytest.fixture
+def mock_formatter():
+    formatter = MagicMock()
+    formatter.format_input.return_value = {"formatted": "input"}
+    return formatter
+
+
+@pytest.fixture
+def mock_parser():
+    parser = MagicMock()
+    parser.parse_output.return_value = RunOutput(
+        output="test output", intermediate_outputs={}
+    )
+    return parser
+
+
 async def test_model_provider_uses_cache(adapter, mock_provider):
     """Test that cached provider is returned if it exists"""
     # Set up cached provider
@@ -197,3 +214,58 @@ async def test_run_strategy(
     # Test
     result = adapter.run_strategy()
     assert result == expected
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "formatter_id,expected_input,expected_calls",
+    [
+        (None, {"original": "input"}, 0),  # No formatter
+        ("test_formatter", {"formatted": "input"}, 1),  # With formatter
+    ],
+)
+async def test_input_formatting(
+    adapter, mock_formatter, mock_parser, formatter_id, expected_input, expected_calls
+):
+    """Test that input formatting is handled correctly based on formatter configuration"""
+    # Mock the model provider to return our formatter ID and parser
+    provider = MagicMock()
+    provider.formatter = formatter_id
+    provider.parser = "test_parser"
+    provider.reasoning_capable = False
+    adapter.model_provider = MagicMock(return_value=provider)
+
+    # Mock the formatter factory and parser factory
+    with (
+        patch(
+            "kiln_ai.adapters.model_adapters.base_adapter.request_formatter_from_id"
+        ) as mock_factory,
+        patch(
+            "kiln_ai.adapters.model_adapters.base_adapter.model_parser_from_id"
+        ) as mock_parser_factory,
+    ):
+        mock_factory.return_value = mock_formatter
+        mock_parser_factory.return_value = mock_parser
+
+        # Mock the _run method to capture the input
+        captured_input = None
+
+        async def mock_run(input):
+            nonlocal captured_input
+            captured_input = input
+            return RunOutput(output="test output", intermediate_outputs={}), None
+
+        adapter._run = mock_run
+
+        # Run the adapter
+        original_input = {"original": "input"}
+        await adapter.invoke_returning_run_output(original_input)
+
+        # Verify formatter was called correctly
+        assert captured_input == expected_input
+        assert mock_factory.call_count == (1 if formatter_id else 0)
+        assert mock_formatter.format_input.call_count == expected_calls
+
+        # Verify original input was preserved in the run
+        if formatter_id:
+            mock_formatter.format_input.assert_called_once_with(original_input)

kiln_ai/adapters/model_adapters/test_litellm_adapter.py

@@ -1,6 +1,7 @@
 import json
 from unittest.mock import Mock, patch
 
+import litellm
 import pytest
 
 from kiln_ai.adapters.ml_model_list import ModelProviderName, StructuredOutputMode
@@ -9,7 +10,7 @@ from kiln_ai.adapters.model_adapters.litellm_adapter import LiteLlmAdapter
 from kiln_ai.adapters.model_adapters.litellm_config import (
     LiteLlmConfig,
 )
-from kiln_ai.datamodel import Project, Task
+from kiln_ai.datamodel import Project, Task, Usage
 
 
 @pytest.fixture
@@ -405,3 +406,66 @@ async def test_build_completion_kwargs(
     # Verify extra body is included
     for key, value in extra_body.items():
         assert kwargs[key] == value
+
+
+@pytest.mark.parametrize(
+    "litellm_usage,cost,expected_usage",
+    [
+        # No usage data
+        (None, None, None),
+        # Only cost
+        (None, 0.5, Usage(cost=0.5)),
+        # Only token counts
+        (
+            litellm.types.utils.Usage(
+                prompt_tokens=10,
+                completion_tokens=20,
+                total_tokens=30,
+            ),
+            None,
+            Usage(input_tokens=10, output_tokens=20, total_tokens=30),
+        ),
+        # Both cost and token counts
+        (
+            litellm.types.utils.Usage(
+                prompt_tokens=10,
+                completion_tokens=20,
+                total_tokens=30,
+            ),
+            0.5,
+            Usage(input_tokens=10, output_tokens=20, total_tokens=30, cost=0.5),
+        ),
+        # Invalid usage type (should be ignored)
+        ({"prompt_tokens": 10}, None, None),
+        # Invalid cost type (should be ignored)
+        (None, "0.5", None),
+    ],
+)
+def test_usage_from_response(config, mock_task, litellm_usage, cost, expected_usage):
+    """Test usage_from_response with various combinations of usage data and cost"""
+    adapter = LiteLlmAdapter(config=config, kiln_task=mock_task)
+
+    # Create a mock response
+    response = Mock(spec=litellm.types.utils.ModelResponse)
+    response.get.return_value = litellm_usage
+    response._hidden_params = {"response_cost": cost}
+
+    # Call the method
+    result = adapter.usage_from_response(response)
+
+    # Verify the result
+    if expected_usage is None:
+        if result is not None:
+            assert result.input_tokens is None
+            assert result.output_tokens is None
+            assert result.total_tokens is None
+            assert result.cost is None
+    else:
+        assert result is not None
+        assert result.input_tokens == expected_usage.input_tokens
+        assert result.output_tokens == expected_usage.output_tokens
+        assert result.total_tokens == expected_usage.total_tokens
+        assert result.cost == expected_usage.cost
+
+    # Verify the response was queried correctly
+    response.get.assert_called_once_with("usage", None)

kiln_ai/adapters/model_adapters/test_saving_adapter_results.py

@@ -11,14 +11,15 @@ from kiln_ai.datamodel import (
     DataSourceType,
     Project,
     Task,
+    Usage,
 )
 from kiln_ai.datamodel.task import RunConfig
 from kiln_ai.utils.config import Config
 
 
 class MockAdapter(BaseAdapter):
-    async def _run(self, input: dict | str) -> dict | str:
-        return RunOutput(output="Test output", intermediate_outputs=None)
+    async def _run(self, input: dict | str) -> tuple[RunOutput, Usage | None]:
+        return RunOutput(output="Test output", intermediate_outputs=None), None
 
     def adapter_name(self) -> str:
         return "mock_adapter"

kiln_ai/adapters/model_adapters/test_structured_output.py

@@ -12,6 +12,7 @@ from kiln_ai.adapters.ml_model_list import (
 from kiln_ai.adapters.model_adapters.base_adapter import (
     BaseAdapter,
     RunOutput,
+    Usage,
 )
 from kiln_ai.adapters.ollama_tools import ollama_online
 from kiln_ai.adapters.test_prompt_adaptors import get_all_models_and_providers
@@ -54,8 +55,8 @@ class MockAdapter(BaseAdapter):
         )
         self.response = response
 
-    async def _run(self, input: str) -> RunOutput:
-        return RunOutput(output=self.response, intermediate_outputs=None)
+    async def _run(self, input: str) -> tuple[RunOutput, Usage | None]:
+        return RunOutput(output=self.response, intermediate_outputs=None), None
 
     def adapter_name(self) -> str:
         return "mock_adapter"
@@ -223,10 +224,7 @@ async def run_structured_input_task(
     with pytest.raises(ValueError):
         # not structured input in dictionary
         await a.invoke("a=1, b=2, c=3")
-    with pytest.raises(
-        ValueError,
-        match="This task requires a specific output schema. While the model produced JSON, that JSON didn't meet the schema.",
-    ):
+    with pytest.raises(ValueError, match="This task requires a specific input"):
         # invalid structured input
         await a.invoke({"a": 1, "b": 2, "d": 3})
 

kiln_ai/adapters/parsers/base_parser.py

@@ -2,9 +2,6 @@ from kiln_ai.adapters.run_output import RunOutput
 
 
 class BaseParser:
-    def __init__(self, structured_output: bool = False):
-        self.structured_output = structured_output
-
     def parse_output(self, original_output: RunOutput) -> RunOutput:
         """
         Method for parsing the output of a model. Typically overridden by subclasses.

kiln_ai/adapters/parsers/parser_registry.py

@@ -6,14 +6,16 @@ from kiln_ai.adapters.parsers.r1_parser import R1ThinkingParser
 from kiln_ai.utils.exhaustive_error import raise_exhaustive_enum_error
 
 
-def model_parser_from_id(parser_id: ModelParserID | None) -> Type[BaseParser]:
+def model_parser_from_id(parser_id: ModelParserID | None) -> BaseParser:
     """
     Get a model parser from its ID.
     """
     match parser_id:
         case None:
-            return BaseParser
+            return BaseParser()
         case ModelParserID.r1_thinking:
-            return R1ThinkingParser
+            return R1ThinkingParser()
+        case ModelParserID.optional_r1_thinking:
+            return R1ThinkingParser(allow_missing_thinking=True)
         case _:
             raise_exhaustive_enum_error(parser_id)

kiln_ai/adapters/parsers/r1_parser.py

@@ -7,6 +7,9 @@ class R1ThinkingParser(BaseParser):
     START_TAG = "<think>"
     END_TAG = "</think>"
 
+    def __init__(self, allow_missing_thinking: bool = False):
+        self.allow_missing_thinking = allow_missing_thinking
+
     def parse_output(self, original_output: RunOutput) -> RunOutput:
         """
         Parse the <think> </think> tags from the response into the intermediate and final outputs.
@@ -27,6 +30,14 @@ class R1ThinkingParser(BaseParser):
             original_output.intermediate_outputs is not None
             and "reasoning" in original_output.intermediate_outputs
         ):
+            # sometimes the output and reasoning are wrapped in newlines
+            if isinstance(original_output.output, str):
+                original_output.output = original_output.output.strip()
+
+            original_output.intermediate_outputs["reasoning"] = (
+                original_output.intermediate_outputs["reasoning"].strip()
+            )
+
             return original_output
 
         # This parser only works for strings
@@ -39,7 +50,10 @@ class R1ThinkingParser(BaseParser):
         # Find the thinking tags
         think_end = cleaned_response.find(self.END_TAG)
         if think_end == -1:
-            raise ValueError("Missing </think> tag")
+            if self.allow_missing_thinking:
+                return original_output
+            else:
+                raise ValueError("Missing </think> tag")
 
         think_tag_start = cleaned_response.find(self.START_TAG)
         if think_tag_start == -1:
@@ -66,7 +80,8 @@ class R1ThinkingParser(BaseParser):
 
         # Add thinking content to intermediate outputs if it exists
         intermediate_outputs = original_output.intermediate_outputs or {}
-        intermediate_outputs["reasoning"] = thinking_content
+        if thinking_content is not None and len(thinking_content) > 0:
+            intermediate_outputs["reasoning"] = thinking_content
 
         return RunOutput(
             output=result,

kiln_ai/adapters/parsers/request_formatters.py

@@ -0,0 +1,40 @@
+import json
+from typing import Dict, Protocol
+
+from kiln_ai.adapters.ml_model_list import ModelFormatterID
+from kiln_ai.utils.exhaustive_error import raise_exhaustive_enum_error
+
+
+class RequestFormatter(Protocol):
+    def format_input(self, original_input: Dict | str) -> Dict | str:
+        """
+        Method for formatting the input to a model.
+        """
+        ...
+
+
+class Qwen3StyleNoThinkFormatter:
+    def format_input(self, original_input: Dict | str) -> Dict | str:
+        """
+        Format the input to a model for Qwen3 /no_think instruction
+        """
+        formatted_input = (
+            original_input
+            if isinstance(original_input, str)
+            else json.dumps(original_input, indent=2)
+        )
+
+        return formatted_input + "\n\n/no_think"
+
+
+def request_formatter_from_id(
+    formatter_id: ModelFormatterID,
+) -> RequestFormatter:
+    """
+    Get a model parser from its ID.
+    """
+    match formatter_id:
+        case ModelFormatterID.qwen3_style_no_think:
+            return Qwen3StyleNoThinkFormatter()
+        case _:
+            raise_exhaustive_enum_error(formatter_id)

kiln_ai/adapters/parsers/test_parser_registry.py

@@ -28,5 +28,5 @@ def test_model_parser_from_id_invalid():
 )
 def test_model_parser_from_id_parametrized(parser_id, expected_class):
     """Test all valid parser IDs using parametrize."""
-    parser_class = model_parser_from_id(parser_id)
-    assert parser_class == expected_class
+    parser = model_parser_from_id(parser_id)
+    assert isinstance(parser, expected_class)

kiln_ai/adapters/parsers/test_r1_parser.py

@@ -46,6 +46,21 @@ def test_response_with_whitespace(parser):
     assert parsed.output.strip() == "This is the result"
 
 
+def test_empty_thinking_content(parser):
+    response = RunOutput(
+        output="""
+        <think>
+
+        </think>
+            This is the result
+    """,
+        intermediate_outputs=None,
+    )
+    parsed = parser.parse_output(response)
+    assert "reasoning" not in parsed.intermediate_outputs
+    assert parsed.output.strip() == "This is the result"
+
+
 def test_missing_start_tag(parser):
     parsed = parser.parse_output(
         RunOutput(output="Some content</think>result", intermediate_outputs=None)
@@ -86,7 +101,7 @@ def test_empty_thinking_content(parser):
         output="<think></think>This is the result", intermediate_outputs=None
     )
     parsed = parser.parse_output(response)
-    assert parsed.intermediate_outputs == {"reasoning": ""}
+    assert "reasoning" not in parsed.intermediate_outputs
     assert parsed.output == "This is the result"
 
 
@@ -154,3 +169,31 @@ def test_intermediate_outputs(parser):
         )
     )
     assert out.intermediate_outputs["reasoning"] == "Some content"
+
+
+def test_strip_newlines(parser):
+    # certain providers via LiteLLM for example, add newlines to the output
+    # and to the reasoning. This tests that we strip those newlines.
+    response = RunOutput(
+        output="\n\nSome content",
+        intermediate_outputs={
+            "reasoning": "\n\nSome thinking\n\n",
+        },
+    )
+    parsed = parser.parse_output(response)
+    assert parsed.output == "Some content"
+    assert parsed.intermediate_outputs["reasoning"] == "Some thinking"
+
+
+def test_strip_newlines_with_structured_output(parser):
+    # certain providers via LiteLLM for example, add newlines to the output
+    # and to the reasoning. This tests that we strip those newlines.
+    response = RunOutput(
+        output={"some_key": "Some content"},
+        intermediate_outputs={
+            "reasoning": "\n\nSome thinking\n\n",
+        },
+    )
+    parsed = parser.parse_output(response)
+    assert parsed.output == {"some_key": "Some content"}
+    assert parsed.intermediate_outputs["reasoning"] == "Some thinking"