letta-nightly 0.11.7.dev20251007104119__py3-none-any.whl → 0.11.7.dev20251008104128__py3-none-any.whl

letta/llm_api/llm_client_base.py

@@ -9,7 +9,7 @@ from openai.types.chat.chat_completion_chunk import ChatCompletionChunk
 from letta.errors import LLMError
 from letta.otel.tracing import log_event, trace_method
 from letta.schemas.embedding_config import EmbeddingConfig
-from letta.schemas.enums import ProviderCategory
+from letta.schemas.enums import AgentType, ProviderCategory
 from letta.schemas.llm_config import LLMConfig
 from letta.schemas.message import Message
 from letta.schemas.openai.chat_completion_response import ChatCompletionResponse
@@ -40,6 +40,7 @@ class LLMClientBase:
     @trace_method
     def send_llm_request(
         self,
+        agent_type: AgentType,
         messages: List[Message],
         llm_config: LLMConfig,
         tools: Optional[List[dict]] = None,  # TODO: change to Tool object
@@ -52,7 +53,7 @@ class LLMClientBase:
         If stream=True, returns a Stream[ChatCompletionChunk] that can be iterated over.
         Otherwise returns a ChatCompletionResponse.
         """
-        request_data = self.build_request_data(messages, llm_config, tools, force_tool_call)
+        request_data = self.build_request_data(agent_type, messages, llm_config, tools, force_tool_call)
 
         try:
             log_event(name="llm_request_sent", attributes=request_data)
@@ -108,6 +109,7 @@ class LLMClientBase:
 
     async def send_llm_batch_request_async(
         self,
+        agent_type: AgentType,
         agent_messages_mapping: Dict[str, List[Message]],
         agent_tools_mapping: Dict[str, List[dict]],
         agent_llm_config_mapping: Dict[str, LLMConfig],
@@ -120,10 +122,12 @@ class LLMClientBase:
     @abstractmethod
     def build_request_data(
         self,
+        agent_type: AgentType,
         messages: List[Message],
         llm_config: LLMConfig,
         tools: List[dict],
         force_tool_call: Optional[str] = None,
+        requires_subsequent_tool_call: bool = False,
     ) -> dict:
         """
         Constructs a request object in the expected data format for this client.

letta/llm_api/openai.py

@@ -40,6 +40,7 @@ from letta.schemas.openai.chat_completion_response import (
     UsageStatistics,
 )
 from letta.schemas.openai.embedding_response import EmbeddingResponse
+from letta.settings import model_settings
 from letta.streaming_interface import AgentChunkStreamingInterface, AgentRefreshStreamingInterface
 from letta.utils import get_tool_call_id, smart_urljoin
 
@@ -82,6 +83,12 @@ def openai_get_model_list(url: str, api_key: Optional[str] = None, fix_url: bool
     headers = {"Content-Type": "application/json"}
     if api_key is not None:
         headers["Authorization"] = f"Bearer {api_key}"
+    # Add optional OpenRouter headers if hitting OpenRouter
+    if "openrouter.ai" in url:
+        if model_settings.openrouter_referer:
+            headers["HTTP-Referer"] = model_settings.openrouter_referer
+        if model_settings.openrouter_title:
+            headers["X-Title"] = model_settings.openrouter_title
 
     logger.debug(f"Sending request to {url}")
     response = None
@@ -139,6 +146,11 @@ async def openai_get_model_list_async(
     headers = {"Content-Type": "application/json"}
     if api_key is not None:
         headers["Authorization"] = f"Bearer {api_key}"
+    if "openrouter.ai" in url:
+        if model_settings.openrouter_referer:
+            headers["HTTP-Referer"] = model_settings.openrouter_referer
+        if model_settings.openrouter_title:
+            headers["X-Title"] = model_settings.openrouter_title
 
     logger.debug(f"Sending request to {url}")
 
@@ -550,7 +562,16 @@ def openai_chat_completions_request_stream(
 
     data = prepare_openai_payload(chat_completion_request)
     data["stream"] = True
-    client = OpenAI(api_key=api_key, base_url=url, max_retries=0)
+    kwargs = {"api_key": api_key, "base_url": url, "max_retries": 0}
+    if "openrouter.ai" in url:
+        headers = {}
+        if model_settings.openrouter_referer:
+            headers["HTTP-Referer"] = model_settings.openrouter_referer
+        if model_settings.openrouter_title:
+            headers["X-Title"] = model_settings.openrouter_title
+        if headers:
+            kwargs["default_headers"] = headers
+    client = OpenAI(**kwargs)
     try:
         stream = client.chat.completions.create(**data)
         for chunk in stream:
@@ -574,7 +595,16 @@ def openai_chat_completions_request(
     https://platform.openai.com/docs/guides/text-generation?lang=curl
     """
     data = prepare_openai_payload(chat_completion_request)
-    client = OpenAI(api_key=api_key, base_url=url, max_retries=0)
+    kwargs = {"api_key": api_key, "base_url": url, "max_retries": 0}
+    if "openrouter.ai" in url:
+        headers = {}
+        if model_settings.openrouter_referer:
+            headers["HTTP-Referer"] = model_settings.openrouter_referer
+        if model_settings.openrouter_title:
+            headers["X-Title"] = model_settings.openrouter_title
+        if headers:
+            kwargs["default_headers"] = headers
+    client = OpenAI(**kwargs)
     log_event(name="llm_request_sent", attributes=data)
     chat_completion = client.chat.completions.create(**data)
     log_event(name="llm_response_received", attributes=chat_completion.model_dump())

letta/llm_api/openai_client.py

@@ -4,10 +4,13 @@ from typing import List, Optional
 
 import openai
 from openai import AsyncOpenAI, AsyncStream, OpenAI
+from openai.types import Reasoning
 from openai.types.chat.chat_completion import ChatCompletion
 from openai.types.chat.chat_completion_chunk import ChatCompletionChunk
+from openai.types.responses import ResponseTextConfigParam
+from openai.types.responses.response_stream_event import ResponseStreamEvent
 
-from letta.constants import LETTA_MODEL_ENDPOINT
+from letta.constants import LETTA_MODEL_ENDPOINT, REQUEST_HEARTBEAT_PARAM
 from letta.errors import (
     ContextWindowExceededError,
     ErrorCode,
@@ -26,6 +29,7 @@ from letta.llm_api.llm_client_base import LLMClientBase
 from letta.local_llm.constants import INNER_THOUGHTS_KWARG, INNER_THOUGHTS_KWARG_DESCRIPTION, INNER_THOUGHTS_KWARG_DESCRIPTION_GO_FIRST
 from letta.log import get_logger
 from letta.otel.tracing import trace_method
+from letta.schemas.agent import AgentType
 from letta.schemas.embedding_config import EmbeddingConfig
 from letta.schemas.letta_message_content import MessageContentType
 from letta.schemas.llm_config import LLMConfig
@@ -38,7 +42,15 @@ from letta.schemas.openai.chat_completion_request import (
     ToolFunctionChoice,
     cast_message_to_subtype,
 )
-from letta.schemas.openai.chat_completion_response import ChatCompletionResponse
+from letta.schemas.openai.chat_completion_response import (
+    ChatCompletionResponse,
+    Choice,
+    FunctionCall,
+    Message as ChoiceMessage,
+    ToolCall,
+    UsageStatistics,
+)
+from letta.schemas.openai.responses_request import ResponsesRequest
 from letta.settings import model_settings
 
 logger = get_logger(__name__)
@@ -117,6 +129,11 @@ def requires_auto_tool_choice(llm_config: LLMConfig) -> bool:
     return False
 
 
+def use_responses_api(llm_config: LLMConfig) -> bool:
+    # TODO can opt in all reasoner models to use the Responses API
+    return is_openai_reasoning_model(llm_config.model)
+
+
 def supports_content_none(llm_config: LLMConfig) -> bool:
     """Certain providers don't support the content None."""
     if "gpt-oss" in llm_config.model:
@@ -128,12 +145,32 @@ class OpenAIClient(LLMClientBase):
     def _prepare_client_kwargs(self, llm_config: LLMConfig) -> dict:
         api_key, _, _ = self.get_byok_overrides(llm_config)
 
+        # Default to global OpenAI key when no BYOK override
         if not api_key:
             api_key = model_settings.openai_api_key or os.environ.get("OPENAI_API_KEY")
-        # supposedly the openai python client requires a dummy API key
-        api_key = api_key or "DUMMY_API_KEY"
+
         kwargs = {"api_key": api_key, "base_url": llm_config.model_endpoint}
 
+        # OpenRouter-specific overrides: use OpenRouter key and optional headers
+        is_openrouter = (llm_config.model_endpoint and "openrouter.ai" in llm_config.model_endpoint) or (
+            llm_config.provider_name == "openrouter"
+        )
+        if is_openrouter:
+            or_key = model_settings.openrouter_api_key or os.environ.get("OPENROUTER_API_KEY")
+            if or_key:
+                kwargs["api_key"] = or_key
+            # Attach optional headers if provided
+            headers = {}
+            if model_settings.openrouter_referer:
+                headers["HTTP-Referer"] = model_settings.openrouter_referer
+            if model_settings.openrouter_title:
+                headers["X-Title"] = model_settings.openrouter_title
+            if headers:
+                kwargs["default_headers"] = headers
+
+        # The OpenAI client requires some API key value
+        kwargs["api_key"] = kwargs.get("api_key") or "DUMMY_API_KEY"
+
         return kwargs
 
     def _prepare_client_kwargs_embedding(self, embedding_config: EmbeddingConfig) -> dict:
@@ -148,10 +185,25 @@ class OpenAIClient(LLMClientBase):
 
         if not api_key:
             api_key = model_settings.openai_api_key or os.environ.get("OPENAI_API_KEY")
-        # supposedly the openai python client requires a dummy API key
-        api_key = api_key or "DUMMY_API_KEY"
         kwargs = {"api_key": api_key, "base_url": llm_config.model_endpoint}
 
+        is_openrouter = (llm_config.model_endpoint and "openrouter.ai" in llm_config.model_endpoint) or (
+            llm_config.provider_name == "openrouter"
+        )
+        if is_openrouter:
+            or_key = model_settings.openrouter_api_key or os.environ.get("OPENROUTER_API_KEY")
+            if or_key:
+                kwargs["api_key"] = or_key
+            headers = {}
+            if model_settings.openrouter_referer:
+                headers["HTTP-Referer"] = model_settings.openrouter_referer
+            if model_settings.openrouter_title:
+                headers["X-Title"] = model_settings.openrouter_title
+            if headers:
+                kwargs["default_headers"] = headers
+
+        kwargs["api_key"] = kwargs.get("api_key") or "DUMMY_API_KEY"
+
         return kwargs
 
     def requires_auto_tool_choice(self, llm_config: LLMConfig) -> bool:
@@ -160,17 +212,182 @@ class OpenAIClient(LLMClientBase):
     def supports_structured_output(self, llm_config: LLMConfig) -> bool:
         return supports_structured_output(llm_config)
 
+    @trace_method
+    def build_request_data_responses(
+        self,
+        agent_type: AgentType,  # if react, use native content + strip heartbeats
+        messages: List[PydanticMessage],
+        llm_config: LLMConfig,
+        tools: Optional[List[dict]] = None,  # Keep as dict for now as per base class
+        force_tool_call: Optional[str] = None,
+        requires_subsequent_tool_call: bool = False,
+    ) -> dict:
+        """
+        Constructs a request object in the expected data format for the OpenAI Responses API.
+        """
+        if llm_config.put_inner_thoughts_in_kwargs:
+            raise ValueError("Inner thoughts in kwargs are not supported for the OpenAI Responses API")
+
+        openai_messages_list = PydanticMessage.to_openai_responses_dicts_from_list(messages)
+        # Add multi-modal support for Responses API by rewriting user messages
+        # into input_text/input_image parts.
+        openai_messages_list = fill_image_content_in_responses_input(openai_messages_list, messages)
+
+        if llm_config.model:
+            model = llm_config.model
+        else:
+            logger.warning(f"Model type not set in llm_config: {llm_config.model_dump_json(indent=4)}")
+            model = None
+
+        # Default to auto, unless there's a forced tool call coming from above or requires_subsequent_tool_call is True
+        tool_choice = None
+        if tools:  # only set tool_choice if tools exist
+            if force_tool_call is not None:
+                tool_choice = {"type": "function", "name": force_tool_call}
+            elif requires_subsequent_tool_call:
+                tool_choice = "required"
+            else:
+                tool_choice = "auto"
+
+        # Convert the tools from the ChatCompletions style to the Responses style
+        if tools:
+            # Get proper typing
+            typed_tools: List[OpenAITool] = [OpenAITool(type="function", function=f) for f in tools]
+
+            # Strip request heartbeat
+            # TODO relax this?
+            if agent_type == AgentType.letta_v1_agent:
+                new_tools = []
+                for tool in typed_tools:
+                    # Remove request_heartbeat from the properties if it exists
+                    if tool.function.parameters and "properties" in tool.function.parameters:
+                        tool.function.parameters["properties"].pop(REQUEST_HEARTBEAT_PARAM, None)
+                        # Also remove from required list if present
+                        if "required" in tool.function.parameters and REQUEST_HEARTBEAT_PARAM in tool.function.parameters["required"]:
+                            tool.function.parameters["required"].remove(REQUEST_HEARTBEAT_PARAM)
+                    new_tools.append(tool.model_copy(deep=True))
+                typed_tools = new_tools
+
+            # Convert to strict mode
+            if supports_structured_output(llm_config):
+                for tool in typed_tools:
+                    try:
+                        structured_output_version = convert_to_structured_output(tool.function.model_dump())
+                        tool.function = FunctionSchema(**structured_output_version)
+                    except ValueError as e:
+                        logger.warning(f"Failed to convert tool function to structured output, tool={tool}, error={e}")
+
+                # Finally convert to a Responses-friendly dict
+                responses_tools = [
+                    {
+                        "type": "function",
+                        "name": t.function.name,
+                        "description": t.function.description,
+                        "parameters": t.function.parameters,
+                        "strict": True,
+                    }
+                    for t in typed_tools
+                ]
+
+            else:
+                # Finally convert to a Responses-friendly dict
+                responses_tools = [
+                    {
+                        "type": "function",
+                        "name": t.function.name,
+                        "description": t.function.description,
+                        "parameters": t.function.parameters,
+                        # "strict": True,
+                    }
+                    for t in typed_tools
+                ]
+        else:
+            responses_tools = None
+
+        # Prepare the request payload
+        data = ResponsesRequest(
+            # Responses specific
+            store=False,
+            include=["reasoning.encrypted_content"],
+            # More or less generic to ChatCompletions API
+            model=model,
+            input=openai_messages_list,
+            tools=responses_tools,
+            tool_choice=tool_choice,
+            max_output_tokens=llm_config.max_tokens,
+            temperature=llm_config.temperature if supports_temperature_param(model) else None,
+            parallel_tool_calls=False,
+        )
+
+        # Add verbosity control for GPT-5 models
+        if supports_verbosity_control(model) and llm_config.verbosity:
+            # data.verbosity = llm_config.verbosity
+            # https://cookbook.openai.com/examples/gpt-5/gpt-5_new_params_and_tools
+            data.text = ResponseTextConfigParam(verbosity=llm_config.verbosity)
+
+        # Add reasoning effort control for reasoning models
+        if is_openai_reasoning_model(model) and llm_config.reasoning_effort:
+            # data.reasoning_effort = llm_config.reasoning_effort
+            data.reasoning = Reasoning(
+                effort=llm_config.reasoning_effort,
+                # NOTE: hardcoding summary level, could put in llm_config?
+                summary="detailed",
+            )
+
+        # TODO I don't see this in Responses?
+        # Add frequency penalty
+        # if llm_config.frequency_penalty is not None:
+        # data.frequency_penalty = llm_config.frequency_penalty
+
+        # Add parallel tool calling
+        if tools and supports_parallel_tool_calling(model):
+            data.parallel_tool_calls = False
+
+        # always set user id for openai requests
+        if self.actor:
+            data.user = self.actor.id
+
+        if llm_config.model_endpoint == LETTA_MODEL_ENDPOINT:
+            if not self.actor:
+                # override user id for inference.letta.com
+                import uuid
+
+                data.user = str(uuid.UUID(int=0))
+
+            data.model = "memgpt-openai"
+
+        request_data = data.model_dump(exclude_unset=True)
+        # print("responses request data", request_data)
+        return request_data
+
     @trace_method
     def build_request_data(
         self,
+        agent_type: AgentType,  # if react, use native content + strip heartbeats
         messages: List[PydanticMessage],
         llm_config: LLMConfig,
         tools: Optional[List[dict]] = None,  # Keep as dict for now as per base class
         force_tool_call: Optional[str] = None,
+        requires_subsequent_tool_call: bool = False,
     ) -> dict:
         """
         Constructs a request object in the expected data format for the OpenAI API.
         """
+        # Shortcut for GPT-5 to use Responses API, but only for letta_v1_agent
+        if use_responses_api(llm_config) and agent_type == AgentType.letta_v1_agent:
+            return self.build_request_data_responses(
+                agent_type=agent_type,
+                messages=messages,
+                llm_config=llm_config,
+                tools=tools,
+                force_tool_call=force_tool_call,
+                requires_subsequent_tool_call=requires_subsequent_tool_call,
+            )
+
+        if agent_type == AgentType.letta_v1_agent:
+            # Safety hard override in case it got set somewhere by accident
+            llm_config.put_inner_thoughts_in_kwargs = False
+
         if tools and llm_config.put_inner_thoughts_in_kwargs:
             # Special case for LM Studio backend since it needs extra guidance to force out the thoughts first
             # TODO(fix)
@@ -208,15 +425,16 @@ class OpenAIClient(LLMClientBase):
         # TODO: This vllm checking is very brittle and is a patch at most
         tool_choice = None
         if tools:  # only set tool_choice if tools exist
-            if self.requires_auto_tool_choice(llm_config):
+            if force_tool_call is not None:
+                tool_choice = ToolFunctionChoice(type="function", function=ToolFunctionChoiceFunctionCall(name=force_tool_call))
+            elif requires_subsequent_tool_call:
+                tool_choice = "required"
+            elif self.requires_auto_tool_choice(llm_config) or agent_type == AgentType.letta_v1_agent:
                 tool_choice = "auto"
             else:
                 # only set if tools is non-Null
                 tool_choice = "required"
 
-            if force_tool_call is not None:
-                tool_choice = ToolFunctionChoice(type="function", function=ToolFunctionChoiceFunctionCall(name=force_tool_call))
-
         if not supports_content_none(llm_config):
             for message in openai_message_list:
                 if message.content is None:
@@ -260,6 +478,21 @@ class OpenAIClient(LLMClientBase):
 
             data.model = "memgpt-openai"
 
+        # For some reason, request heartbeats are still leaking into here...
+        # So strip them manually for v3
+        if agent_type == AgentType.letta_v1_agent:
+            new_tools = []
+            if data.tools:
+                for tool in data.tools:
+                    # Remove request_heartbeat from the properties if it exists
+                    if tool.function.parameters and "properties" in tool.function.parameters:
+                        tool.function.parameters["properties"].pop(REQUEST_HEARTBEAT_PARAM, None)
+                        # Also remove from required list if present
+                        if "required" in tool.function.parameters and REQUEST_HEARTBEAT_PARAM in tool.function.parameters["required"]:
+                            tool.function.parameters["required"].remove(REQUEST_HEARTBEAT_PARAM)
+                    new_tools.append(tool.model_copy(deep=True))
+                data.tools = new_tools
+
         if data.tools is not None and len(data.tools) > 0:
             # Convert to structured output style (which has 'strict' and no optionals)
             for tool in data.tools:
@@ -270,6 +503,14 @@ class OpenAIClient(LLMClientBase):
                     except ValueError as e:
                         logger.warning(f"Failed to convert tool function to structured output, tool={tool}, error={e}")
         request_data = data.model_dump(exclude_unset=True)
+
+        # If Ollama
+        # if llm_config.handle.startswith("ollama/") and llm_config.enable_reasoner:
+        # Sadly, reasoning via the OpenAI proxy on Ollama only works for Harmony/gpt-oss
+        # Ollama's OpenAI layer simply looks for the presence of 'reasoining' or 'reasoning_effort'
+        # If set, then in the backend "medium" thinking is turned on
+        # request_data["reasoning_effort"] = "medium"
+
         return request_data
 
     @trace_method
@@ -278,8 +519,13 @@ class OpenAIClient(LLMClientBase):
         Performs underlying synchronous request to OpenAI API and returns raw response dict.
         """
         client = OpenAI(**self._prepare_client_kwargs(llm_config))
-        response: ChatCompletion = client.chat.completions.create(**request_data)
-        return response.model_dump()
+        # Route based on payload shape: Responses uses 'input', Chat Completions uses 'messages'
+        if "input" in request_data and "messages" not in request_data:
+            resp = client.responses.create(**request_data)
+            return resp.model_dump()
+        else:
+            response: ChatCompletion = client.chat.completions.create(**request_data)
+            return response.model_dump()
 
     @trace_method
     async def request_async(self, request_data: dict, llm_config: LLMConfig) -> dict:
@@ -288,8 +534,13 @@ class OpenAIClient(LLMClientBase):
         """
         kwargs = await self._prepare_client_kwargs_async(llm_config)
         client = AsyncOpenAI(**kwargs)
-        response: ChatCompletion = await client.chat.completions.create(**request_data)
-        return response.model_dump()
+        # Route based on payload shape: Responses uses 'input', Chat Completions uses 'messages'
+        if "input" in request_data and "messages" not in request_data:
+            resp = await client.responses.create(**request_data)
+            return resp.model_dump()
+        else:
+            response: ChatCompletion = await client.chat.completions.create(**request_data)
+            return response.model_dump()
 
     def is_reasoning_model(self, llm_config: LLMConfig) -> bool:
         return is_openai_reasoning_model(llm_config.model)
@@ -305,6 +556,84 @@ class OpenAIClient(LLMClientBase):
         Converts raw OpenAI response dict into the ChatCompletionResponse Pydantic model.
         Handles potential extraction of inner thoughts if they were added via kwargs.
         """
+        if "object" in response_data and response_data["object"] == "response":
+            # Map Responses API shape to Chat Completions shape
+            # See example payload in tests/integration_test_send_message_v2.py
+            model = response_data.get("model")
+
+            # Extract usage
+            usage = response_data.get("usage", {}) or {}
+            prompt_tokens = usage.get("input_tokens") or 0
+            completion_tokens = usage.get("output_tokens") or 0
+            total_tokens = usage.get("total_tokens") or (prompt_tokens + completion_tokens)
+
+            # Extract assistant message text from the outputs list
+            outputs = response_data.get("output") or []
+            assistant_text_parts = []
+            reasoning_summary_parts = None
+            reasoning_content_signature = None
+            tool_calls = None
+            finish_reason = "stop" if (response_data.get("status") == "completed") else None
+
+            # Optionally capture reasoning presence
+            found_reasoning = False
+            for out in outputs:
+                out_type = (out or {}).get("type")
+                if out_type == "message":
+                    content_list = (out or {}).get("content") or []
+                    for part in content_list:
+                        if (part or {}).get("type") == "output_text":
+                            text_val = (part or {}).get("text")
+                            if text_val:
+                                assistant_text_parts.append(text_val)
+                elif out_type == "reasoning":
+                    found_reasoning = True
+                    reasoning_summary_parts = [part.get("text") for part in out.get("summary")]
+                    reasoning_content_signature = out.get("encrypted_content")
+                elif out_type == "function_call":
+                    tool_calls = [
+                        ToolCall(
+                            id=out.get("call_id"),
+                            type="function",
+                            function=FunctionCall(
+                                name=out.get("name"),
+                                arguments=out.get("arguments"),
+                            ),
+                        )
+                    ]
+
+            assistant_text = "\n".join(assistant_text_parts) if assistant_text_parts else None
+
+            # Build ChatCompletionResponse-compatible structure
+            # Imports for these Pydantic models are already present in this module
+            choice = Choice(
+                index=0,
+                finish_reason=finish_reason,
+                message=ChoiceMessage(
+                    role="assistant",
+                    content=assistant_text or "",
+                    reasoning_content="\n".join(reasoning_summary_parts) if reasoning_summary_parts else None,
+                    reasoning_content_signature=reasoning_content_signature if reasoning_summary_parts else None,
+                    redacted_reasoning_content=None,
+                    omitted_reasoning_content=False,
+                    tool_calls=tool_calls,
+                ),
+            )
+
+            chat_completion_response = ChatCompletionResponse(
+                id=response_data.get("id", ""),
+                choices=[choice],
+                created=int(response_data.get("created_at") or 0),
+                model=model or (llm_config.model if hasattr(llm_config, "model") else None),
+                usage=UsageStatistics(
+                    prompt_tokens=prompt_tokens,
+                    completion_tokens=completion_tokens,
+                    total_tokens=total_tokens,
+                ),
+            )
+
+            return chat_completion_response
+
         # OpenAI's response structure directly maps to ChatCompletionResponse
         # We just need to instantiate the Pydantic model for validation and type safety.
         chat_completion_response = ChatCompletionResponse(**response_data)
@@ -322,15 +651,36 @@ class OpenAIClient(LLMClientBase):
         return chat_completion_response
 
     @trace_method
-    async def stream_async(self, request_data: dict, llm_config: LLMConfig) -> AsyncStream[ChatCompletionChunk]:
+    async def stream_async(self, request_data: dict, llm_config: LLMConfig) -> AsyncStream[ChatCompletionChunk | ResponseStreamEvent]:
         """
         Performs underlying asynchronous streaming request to OpenAI and returns the async stream iterator.
         """
         kwargs = await self._prepare_client_kwargs_async(llm_config)
         client = AsyncOpenAI(**kwargs)
-        response_stream: AsyncStream[ChatCompletionChunk] = await client.chat.completions.create(
-            **request_data, stream=True, stream_options={"include_usage": True}
-        )
+
+        # Route based on payload shape: Responses uses 'input', Chat Completions uses 'messages'
+        if "input" in request_data and "messages" not in request_data:
+            response_stream: AsyncStream[ResponseStreamEvent] = await client.responses.create(
+                **request_data,
+                stream=True,
+                # stream_options={"include_usage": True},
+            )
+        else:
+            response_stream: AsyncStream[ChatCompletionChunk] = await client.chat.completions.create(
+                **request_data,
+                stream=True,
+                stream_options={"include_usage": True},
+            )
+        return response_stream
+
+    @trace_method
+    async def stream_async_responses(self, request_data: dict, llm_config: LLMConfig) -> AsyncStream[ResponseStreamEvent]:
+        """
+        Performs underlying asynchronous streaming request to OpenAI and returns the async stream iterator.
+        """
+        kwargs = await self._prepare_client_kwargs_async(llm_config)
+        client = AsyncOpenAI(**kwargs)
+        response_stream: AsyncStream[ResponseStreamEvent] = await client.responses.create(**request_data, stream=True)
         return response_stream
 
     @trace_method
@@ -539,3 +889,58 @@ def fill_image_content_in_messages(openai_message_list: List[dict], pydantic_mes
         new_message_list.append({"role": "user", "content": message_content})
 
     return new_message_list
+
+
+def fill_image_content_in_responses_input(openai_message_list: List[dict], pydantic_message_list: List[PydanticMessage]) -> List[dict]:
+    """
+    Rewrite user messages in the Responses API input to embed multi-modal parts inside
+    the message's content array (not as top-level items).
+
+    Expected structure for Responses API input messages:
+      { "type": "message", "role": "user", "content": [
+           {"type": "input_text", "text": "..."},
+           {"type": "input_image", "image_url": {"url": "data:<mime>;base64,<data>", "detail": "auto"}}
+         ] }
+
+    Non-user items are left unchanged.
+    """
+    user_msgs = [m for m in pydantic_message_list if getattr(m, "role", None) == "user"]
+    user_idx = 0
+
+    rewritten: List[dict] = []
+    for item in openai_message_list:
+        if isinstance(item, dict) and item.get("role") == "user":
+            if user_idx >= len(user_msgs):
+                rewritten.append(item)
+                continue
+
+            pm = user_msgs[user_idx]
+            user_idx += 1
+
+            # Only rewrite if the pydantic message actually contains multiple parts or images
+            if not isinstance(pm.content, list) or (len(pm.content) == 1 and pm.content[0].type == MessageContentType.text):
+                rewritten.append(item)
+                continue
+
+            parts: List[dict] = []
+            for content in pm.content:
+                if content.type == MessageContentType.text:
+                    parts.append({"type": "input_text", "text": content.text})
+                elif content.type == MessageContentType.image:
+                    # For Responses API, image_url is a string and detail is required
+                    data_url = f"data:{content.source.media_type};base64,{content.source.data}"
+                    parts.append(
+                        {"type": "input_image", "image_url": data_url, "detail": getattr(content.source, "detail", None) or "auto"}
+                    )
+                else:
+                    # Skip unsupported content types for Responses input
+                    continue
+
+            # Update message content to include multi-modal parts (EasyInputMessageParam style)
+            new_item = dict(item)
+            new_item["content"] = parts
+            rewritten.append(new_item)
+        else:
+            rewritten.append(item)
+
+    return rewritten