llama-index-llms-openai 0.3.41__py3-none-any.whl → 0.3.43__py3-none-any.whl

llama_index/llms/openai/base.py

@@ -909,7 +909,7 @@ class OpenAI(FunctionCallingLLM):
         **kwargs: Any,
     ) -> Dict[str, Any]:
         """Predict and call the tool."""
-        tool_specs = [tool.metadata.to_openai_tool() for tool in tools]
+        tool_specs = [tool.metadata.to_openai_tool(skip_length_check=True) for tool in tools]
 
         # if strict is passed in, use, else default to the class-level attribute, else default to True`
         if strict is not None:

llama_index/llms/openai/responses.py

@@ -1,6 +1,7 @@
 import functools
 import httpx
 import tiktoken
+import base64
 from openai import AsyncOpenAI, AzureOpenAI
 from openai import OpenAI as SyncOpenAI
 from openai.types.responses import (
@@ -23,7 +24,10 @@ from openai.types.responses import (
     ResponseFunctionWebSearch,
     ResponseComputerToolCall,
     ResponseReasoningItem,
+    ResponseCodeInterpreterToolCall,
+    ResponseImageGenCallPartialImageEvent,
 )
+from openai.types.responses.response_output_item import ImageGenerationCall, McpCall
 from typing import (
     TYPE_CHECKING,
     Any,
@@ -59,7 +63,9 @@ from llama_index.core.base.llms.types import (
     CompletionResponseGen,
     LLMMetadata,
     MessageRole,
+    ContentBlock,
     TextBlock,
+    ImageBlock,
 )
 from llama_index.core.bridge.pydantic import (
     Field,
@@ -445,9 +451,9 @@ class OpenAIResponses(FunctionCallingLLM):
         message = ChatMessage(role=MessageRole.ASSISTANT, blocks=[])
         additional_kwargs = {"built_in_tool_calls": []}
         tool_calls = []
+        blocks: List[ContentBlock] = []
         for item in output:
             if isinstance(item, ResponseOutputMessage):
-                blocks = []
                 for part in item.content:
                     if hasattr(part, "text"):
                         blocks.append(TextBlock(text=part.text))
@@ -457,6 +463,17 @@ class OpenAIResponses(FunctionCallingLLM):
                         additional_kwargs["refusal"] = part.refusal
 
                 message.blocks.extend(blocks)
+            elif isinstance(item, ImageGenerationCall):
+                # return an ImageBlock if there is image generation
+                if item.status != "failed":
+                    additional_kwargs["built_in_tool_calls"].append(item)
+                    if item.result is not None:
+                        image_bytes = base64.b64decode(item.result)
+                        blocks.append(ImageBlock(image=image_bytes))
+            elif isinstance(item, ResponseCodeInterpreterToolCall):
+                additional_kwargs["built_in_tool_calls"].append(item)
+            elif isinstance(item, McpCall):
+                additional_kwargs["built_in_tool_calls"].append(item)
             elif isinstance(item, ResponseFileSearchToolCall):
                 additional_kwargs["built_in_tool_calls"].append(item)
             elif isinstance(item, ResponseFunctionToolCall):
@@ -475,6 +492,7 @@ class OpenAIResponses(FunctionCallingLLM):
 
     @llm_retry_decorator
     def _chat(self, messages: Sequence[ChatMessage], **kwargs: Any) -> ChatResponse:
+        kwargs_dict = self._get_model_kwargs(**kwargs)
         message_dicts = to_openai_message_dicts(
             messages,
             model=self.model,
@@ -484,7 +502,7 @@ class OpenAIResponses(FunctionCallingLLM):
         response: Response = self._client.responses.create(
             input=message_dicts,
             stream=False,
-            **self._get_model_kwargs(**kwargs),
+            **kwargs_dict,
         )
 
         if self.track_previous_responses:
@@ -499,7 +517,6 @@ class OpenAIResponses(FunctionCallingLLM):
     @staticmethod
     def process_response_event(
         event: ResponseStreamEvent,
-        content: str,
         tool_calls: List[ResponseFunctionToolCall],
         built_in_tool_calls: List[Any],
         additional_kwargs: Dict[str, Any],
@@ -507,7 +524,7 @@ class OpenAIResponses(FunctionCallingLLM):
         track_previous_responses: bool,
         previous_response_id: Optional[str] = None,
     ) -> Tuple[
-        str,
+        List[ContentBlock],
         List[ResponseFunctionToolCall],
         List[Any],
         Dict[str, Any],
@@ -534,7 +551,8 @@ class OpenAIResponses(FunctionCallingLLM):
         """
         delta = ""
         updated_previous_response_id = previous_response_id
-
+        # we use blocks instead of content, since now we also support images! :)
+        blocks: List[ContentBlock] = []
         if isinstance(event, ResponseCreatedEvent) or isinstance(
             event, ResponseInProgressEvent
         ):
@@ -548,7 +566,11 @@ class OpenAIResponses(FunctionCallingLLM):
         elif isinstance(event, ResponseTextDeltaEvent):
             # Text content is being added
             delta = event.delta
-            content += delta
+            blocks.append(TextBlock(text=delta))
+        elif isinstance(event, ResponseImageGenCallPartialImageEvent):
+            # Partial image
+            if event.partial_image_b64:
+                blocks.append(ImageBlock(image=base64.b64decode(event.partial_image_b64), detail=f"id_{event.partial_image_index}"))
         elif isinstance(event, ResponseFunctionCallArgumentsDeltaEvent):
             # Function call arguments are being streamed
             if current_tool_call is not None:
@@ -586,7 +608,7 @@ class OpenAIResponses(FunctionCallingLLM):
                 additional_kwargs["usage"] = event.response.usage
 
         return (
-            content,
+            blocks,
             tool_calls,
             built_in_tool_calls,
             additional_kwargs,
@@ -606,7 +628,6 @@ class OpenAIResponses(FunctionCallingLLM):
         )
 
         def gen() -> ChatResponseGen:
-            content = ""
             tool_calls = []
             built_in_tool_calls = []
             additional_kwargs = {"built_in_tool_calls": []}
@@ -620,7 +641,7 @@ class OpenAIResponses(FunctionCallingLLM):
             ):
                 # Process the event and update state
                 (
-                    content,
+                    blocks,
                     tool_calls,
                     built_in_tool_calls,
                     additional_kwargs,
@@ -629,7 +650,6 @@ class OpenAIResponses(FunctionCallingLLM):
                     delta,
                 ) = OpenAIResponses.process_response_event(
                     event=event,
-                    content=content,
                     tool_calls=tool_calls,
                     built_in_tool_calls=built_in_tool_calls,
                     additional_kwargs=additional_kwargs,
@@ -651,7 +671,7 @@ class OpenAIResponses(FunctionCallingLLM):
                 yield ChatResponse(
                     message=ChatMessage(
                         role=MessageRole.ASSISTANT,
-                        content=content,
+                        blocks=blocks,
                         additional_kwargs={"tool_calls": tool_calls}
                         if tool_calls
                         else {},
@@ -732,7 +752,6 @@ class OpenAIResponses(FunctionCallingLLM):
         )
 
         async def gen() -> ChatResponseAsyncGen:
-            content = ""
             tool_calls = []
             built_in_tool_calls = []
             additional_kwargs = {"built_in_tool_calls": []}
@@ -748,7 +767,7 @@ class OpenAIResponses(FunctionCallingLLM):
             async for event in response_stream:
                 # Process the event and update state
                 (
-                    content,
+                    blocks,
                     tool_calls,
                     built_in_tool_calls,
                     additional_kwargs,
@@ -757,7 +776,6 @@ class OpenAIResponses(FunctionCallingLLM):
                     delta,
                 ) = OpenAIResponses.process_response_event(
                     event=event,
-                    content=content,
                     tool_calls=tool_calls,
                     built_in_tool_calls=built_in_tool_calls,
                     additional_kwargs=additional_kwargs,
@@ -779,7 +797,7 @@ class OpenAIResponses(FunctionCallingLLM):
                 yield ChatResponse(
                     message=ChatMessage(
                         role=MessageRole.ASSISTANT,
-                        content=content,
+                        blocks=blocks,
                         additional_kwargs={"tool_calls": tool_calls}
                         if tool_calls
                         else {},

llama_index/llms/openai/utils.py

@@ -411,7 +411,7 @@ def to_openai_responses_message_dict(
     message: ChatMessage,
     drop_none: bool = False,
     model: Optional[str] = None,
-) -> Union[Dict[str, Any], List[Dict[str, Any]]]:
+) -> Union[str, Dict[str, Any], List[Dict[str, Any]]]:
     """Convert a ChatMessage to an OpenAI message dict."""
     content = []
     content_txt = ""
@@ -427,7 +427,13 @@ def to_openai_responses_message_dict(
                 mimetype = block._guess_mimetype()
             else:
                 b64_string = block.data.decode("utf-8")
-            content.append({"type": "input_file", "filename": block.title, "file_data": f"data:{mimetype};base64,{b64_string}"})
+            content.append(
+                {
+                    "type": "input_file",
+                    "filename": block.title,
+                    "file_data": f"data:{mimetype};base64,{b64_string}",
+                }
+            )
         elif isinstance(block, ImageBlock):
             if block.url:
                 content.append(
@@ -492,6 +498,16 @@ def to_openai_responses_message_dict(
         ]
 
         return message_dicts
+
+    # there are some cases (like image generation or MCP tool call) that only support the string input
+    # this is why, if context_txt is a non-empty string, all the blocks are TextBlocks and the role is user, we return directly context_txt
+    elif (
+        isinstance(content_txt, str)
+        and len(content_txt) != 0
+        and all(item["type"] == "input_text" for item in content)
+        and message.role.value == "user"
+    ):
+        return content_txt
     else:
         message_dict = {
             "role": message.role.value,
@@ -526,10 +542,11 @@ def to_openai_message_dicts(
     drop_none: bool = False,
     model: Optional[str] = None,
     is_responses_api: bool = False,
-) -> List[ChatCompletionMessageParam]:
+) -> Union[List[ChatCompletionMessageParam], str]:
     """Convert generic messages to OpenAI message dicts."""
     if is_responses_api:
         final_message_dicts = []
+        final_message_txt = ""
         for message in messages:
             message_dicts = to_openai_responses_message_dict(
                 message,
@@ -538,9 +555,13 @@ def to_openai_message_dicts(
             )
             if isinstance(message_dicts, list):
                 final_message_dicts.extend(message_dicts)
+            elif isinstance(message_dicts, str):
+                final_message_txt += message_dicts
             else:
                 final_message_dicts.append(message_dicts)
-
+        # this follows the logic of having a string-only input from to_openai_responses_message_dict
+        if final_message_txt:
+            return final_message_txt
         return final_message_dicts
     else:
         return [

{llama_index_llms_openai-0.3.41.dist-info → llama_index_llms_openai-0.3.43.dist-info}/METADATA

@@ -1,13 +1,13 @@
 Metadata-Version: 2.4
 Name: llama-index-llms-openai
-Version: 0.3.41
+Version: 0.3.43
 Summary: llama-index llms openai integration
 Author: llama-index
 License-Expression: MIT
 License-File: LICENSE
 Requires-Python: <4.0,>=3.9
-Requires-Dist: llama-index-core<0.13,>=0.12.36
-Requires-Dist: openai<2,>=1.66.3
+Requires-Dist: llama-index-core<0.13,>=0.12.37
+Requires-Dist: openai<2,>=1.81.0
 Description-Content-Type: text/markdown
 
 # LlamaIndex Llms Integration: Openai

llama_index_llms_openai-0.3.43.dist-info/RECORD

@@ -0,0 +1,9 @@
+llama_index/llms/openai/__init__.py,sha256=8nmgixeXifQ4eVSgtCic54WxXqrrpXQPL4rhACWCSFs,229
+llama_index/llms/openai/base.py,sha256=2ojLUicnyDwVXO8-_kis93Yibrc8tQoUthSsY7tSyn8,38642
+llama_index/llms/openai/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+llama_index/llms/openai/responses.py,sha256=SLRLbmwNg7aKVV21l0EUXL4vfytltOaDHgIzg8rysEQ,35737
+llama_index/llms/openai/utils.py,sha256=K8LQ6ykuU-TPzIJQLvZ869q5aGvXSM9HAJIX8GQfM_s,27913
+llama_index_llms_openai-0.3.43.dist-info/METADATA,sha256=dGug9DX9IRmSogvGlqir61V5OWWF4yjgaGAtvkNE0CA,3040
+llama_index_llms_openai-0.3.43.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+llama_index_llms_openai-0.3.43.dist-info/licenses/LICENSE,sha256=JPQLUZD9rKvCTdu192Nk0V5PAwklIg6jANii3UmTyMs,1065
+llama_index_llms_openai-0.3.43.dist-info/RECORD,,

llama_index_llms_openai-0.3.41.dist-info/RECORD

@@ -1,9 +0,0 @@
-llama_index/llms/openai/__init__.py,sha256=8nmgixeXifQ4eVSgtCic54WxXqrrpXQPL4rhACWCSFs,229
-llama_index/llms/openai/base.py,sha256=fu4Y1uX8IuOtVTHqaG-tyd_PpN9bUfnW6ESoyaptaVs,38620
-llama_index/llms/openai/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-llama_index/llms/openai/responses.py,sha256=SdTAqO2FqGuvm9u01BlI92IpY0RhOaJ3mU03Rc0OHso,34525
-llama_index/llms/openai/utils.py,sha256=BiHTTRgCQ07GUhuWxG_J_vG8-YK3FIYgRjg2cwix2Qk,27022
-llama_index_llms_openai-0.3.41.dist-info/METADATA,sha256=Xfj-rmVx63Ezz_PbvtHLgMHaqLWx8nmKqdRLw0GH_c0,3040
-llama_index_llms_openai-0.3.41.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-llama_index_llms_openai-0.3.41.dist-info/licenses/LICENSE,sha256=JPQLUZD9rKvCTdu192Nk0V5PAwklIg6jANii3UmTyMs,1065
-llama_index_llms_openai-0.3.41.dist-info/RECORD,,