llama-index-llms-openai 0.6.3__tar.gz → 0.6.5__tar.gz

{llama_index_llms_openai-0.6.3 → llama_index_llms_openai-0.6.5}/PKG-INFO

@@ -1,12 +1,12 @@
 Metadata-Version: 2.4
 Name: llama-index-llms-openai
-Version: 0.6.3
+Version: 0.6.5
 Summary: llama-index llms openai integration
 Author: llama-index
 License-Expression: MIT
 License-File: LICENSE
 Requires-Python: <4.0,>=3.9
-Requires-Dist: llama-index-core<0.15,>=0.14.3
+Requires-Dist: llama-index-core<0.15,>=0.14.5
 Requires-Dist: openai<2,>=1.108.1
 Description-Content-Type: text/markdown
 

{llama_index_llms_openai-0.6.3 → llama_index_llms_openai-0.6.5}/llama_index/llms/openai/base.py

@@ -43,6 +43,8 @@ from llama_index.core.base.llms.types import (
     CompletionResponseGen,
     LLMMetadata,
     MessageRole,
+    ToolCallBlock,
+    TextBlock,
 )
 from llama_index.core.bridge.pydantic import (
     Field,
@@ -121,9 +123,15 @@ class Tokenizer(Protocol):
 
 
 def force_single_tool_call(response: ChatResponse) -> None:
-    tool_calls = response.message.additional_kwargs.get("tool_calls", [])
+    tool_calls = [
+        block for block in response.message.blocks if isinstance(block, ToolCallBlock)
+    ]
     if len(tool_calls) > 1:
-        response.message.additional_kwargs["tool_calls"] = [tool_calls[0]]
+        response.message.blocks = [
+            block
+            for block in response.message.blocks
+            if not isinstance(block, ToolCallBlock)
+        ] + [tool_calls[0]]
 
 
 class OpenAI(FunctionCallingLLM):
@@ -528,6 +536,7 @@ class OpenAI(FunctionCallingLLM):
                 messages=message_dicts,
                 **self._get_model_kwargs(stream=True, **kwargs),
             ):
+                blocks = []
                 response = cast(ChatCompletionChunk, response)
                 if len(response.choices) > 0:
                     delta = response.choices[0].delta
@@ -545,17 +554,27 @@ class OpenAI(FunctionCallingLLM):
                 role = delta.role or MessageRole.ASSISTANT
                 content_delta = delta.content or ""
                 content += content_delta
+                blocks.append(TextBlock(text=content))
 
                 additional_kwargs = {}
                 if is_function:
                     tool_calls = update_tool_calls(tool_calls, delta.tool_calls)
                     if tool_calls:
                         additional_kwargs["tool_calls"] = tool_calls
+                        for tool_call in tool_calls:
+                            if tool_call.function:
+                                blocks.append(
+                                    ToolCallBlock(
+                                        tool_call_id=tool_call.id,
+                                        tool_kwargs=tool_call.function.arguments or {},
+                                        tool_name=tool_call.function.name or "",
+                                    )
+                                )
 
                 yield ChatResponse(
                     message=ChatMessage(
                         role=role,
-                        content=content,
+                        blocks=blocks,
                         additional_kwargs=additional_kwargs,
                     ),
                     delta=content_delta,
@@ -785,6 +804,7 @@ class OpenAI(FunctionCallingLLM):
                 messages=message_dicts,
                 **self._get_model_kwargs(stream=True, **kwargs),
             ):
+                blocks = []
                 response = cast(ChatCompletionChunk, response)
                 if len(response.choices) > 0:
                     # check if the first chunk has neither content nor tool_calls
@@ -812,17 +832,27 @@ class OpenAI(FunctionCallingLLM):
                 role = delta.role or MessageRole.ASSISTANT
                 content_delta = delta.content or ""
                 content += content_delta
+                blocks.append(TextBlock(text=content))
 
                 additional_kwargs = {}
                 if is_function:
                     tool_calls = update_tool_calls(tool_calls, delta.tool_calls)
                     if tool_calls:
                         additional_kwargs["tool_calls"] = tool_calls
+                        for tool_call in tool_calls:
+                            if tool_call.function:
+                                blocks.append(
+                                    ToolCallBlock(
+                                        tool_call_id=tool_call.id,
+                                        tool_kwargs=tool_call.function.arguments or {},
+                                        tool_name=tool_call.function.name or "",
+                                    )
+                                )
 
                 yield ChatResponse(
                     message=ChatMessage(
                         role=role,
-                        content=content,
+                        blocks=blocks,
                         additional_kwargs=additional_kwargs,
                     ),
                     delta=content_delta,
@@ -960,36 +990,71 @@ class OpenAI(FunctionCallingLLM):
         **kwargs: Any,
     ) -> List[ToolSelection]:
         """Predict and call the tool."""
-        tool_calls = response.message.additional_kwargs.get("tool_calls", [])
-
-        if len(tool_calls) < 1:
-            if error_on_no_tool_call:
-                raise ValueError(
-                    f"Expected at least one tool call, but got {len(tool_calls)} tool calls."
+        tool_calls = [
+            block
+            for block in response.message.blocks
+            if isinstance(block, ToolCallBlock)
+        ]
+        if tool_calls:
+            if len(tool_calls) < 1:
+                if error_on_no_tool_call:
+                    raise ValueError(
+                        f"Expected at least one tool call, but got {len(tool_calls)} tool calls."
+                    )
+                else:
+                    return []
+
+            tool_selections = []
+            for tool_call in tool_calls:
+                # this should handle both complete and partial jsons
+                try:
+                    if isinstance(tool_call.tool_kwargs, str):
+                        argument_dict = parse_partial_json(tool_call.tool_kwargs)
+                    else:
+                        argument_dict = tool_call.tool_kwargs
+                except (ValueError, TypeError, JSONDecodeError):
+                    argument_dict = {}
+
+                tool_selections.append(
+                    ToolSelection(
+                        tool_id=tool_call.tool_call_id or "",
+                        tool_name=tool_call.tool_name,
+                        tool_kwargs=argument_dict,
+                    )
                 )
-            else:
-                return []
 
-        tool_selections = []
-        for tool_call in tool_calls:
-            if tool_call.type != "function":
-                raise ValueError("Invalid tool type. Unsupported by OpenAI llm")
+            return tool_selections
+        else:  # keep it backward-compatible
+            tool_calls = response.message.additional_kwargs.get("tool_calls", [])
 
-            # this should handle both complete and partial jsons
-            try:
-                argument_dict = parse_partial_json(tool_call.function.arguments)
-            except (ValueError, TypeError, JSONDecodeError):
-                argument_dict = {}
-
-            tool_selections.append(
-                ToolSelection(
-                    tool_id=tool_call.id,
-                    tool_name=tool_call.function.name,
-                    tool_kwargs=argument_dict,
+            if len(tool_calls) < 1:
+                if error_on_no_tool_call:
+                    raise ValueError(
+                        f"Expected at least one tool call, but got {len(tool_calls)} tool calls."
+                    )
+                else:
+                    return []
+
+            tool_selections = []
+            for tool_call in tool_calls:
+                if tool_call.type != "function":
+                    raise ValueError("Invalid tool type. Unsupported by OpenAI llm")
+
+                # this should handle both complete and partial jsons
+                try:
+                    argument_dict = parse_partial_json(tool_call.function.arguments)
+                except (ValueError, TypeError, JSONDecodeError):
+                    argument_dict = {}
+
+                tool_selections.append(
+                    ToolSelection(
+                        tool_id=tool_call.id,
+                        tool_name=tool_call.function.name,
+                        tool_kwargs=argument_dict,
+                    )
                 )
-            )
 
-        return tool_selections
+            return tool_selections
 
     def _prepare_schema(
         self, llm_kwargs: Optional[Dict[str, Any]], output_cls: Type[Model]

{llama_index_llms_openai-0.6.3 → llama_index_llms_openai-0.6.5}/llama_index/llms/openai/responses.py

@@ -44,6 +44,7 @@ from typing import (
     Type,
     Union,
     runtime_checkable,
+    cast,
 )
 
 import llama_index.core.instrumentation as instrument
@@ -67,6 +68,7 @@ from llama_index.core.base.llms.types import (
     TextBlock,
     ImageBlock,
     ThinkingBlock,
+    ToolCallBlock,
 )
 from llama_index.core.bridge.pydantic import (
     Field,
@@ -131,9 +133,15 @@ class Tokenizer(Protocol):
 
 
 def force_single_tool_call(response: ChatResponse) -> None:
-    tool_calls = response.message.additional_kwargs.get("tool_calls", [])
+    tool_calls = [
+        block for block in response.message.blocks if isinstance(block, ToolCallBlock)
+    ]
     if len(tool_calls) > 1:
-        response.message.additional_kwargs["tool_calls"] = [tool_calls[0]]
+        response.message.blocks = [
+            block
+            for block in response.message.blocks
+            if not isinstance(block, ToolCallBlock)
+        ] + [tool_calls[0]]
 
 
 class OpenAIResponses(FunctionCallingLLM):
@@ -454,7 +462,6 @@ class OpenAIResponses(FunctionCallingLLM):
     def _parse_response_output(output: List[ResponseOutputItem]) -> ChatResponse:
         message = ChatMessage(role=MessageRole.ASSISTANT, blocks=[])
         additional_kwargs = {"built_in_tool_calls": []}
-        tool_calls = []
         blocks: List[ContentBlock] = []
         for item in output:
             if isinstance(item, ResponseOutputMessage):
@@ -481,7 +488,13 @@ class OpenAIResponses(FunctionCallingLLM):
             elif isinstance(item, ResponseFileSearchToolCall):
                 additional_kwargs["built_in_tool_calls"].append(item)
             elif isinstance(item, ResponseFunctionToolCall):
-                tool_calls.append(item)
+                message.blocks.append(
+                    ToolCallBlock(
+                        tool_name=item.name,
+                        tool_call_id=item.call_id,
+                        tool_kwargs=item.arguments,
+                    )
+                )
             elif isinstance(item, ResponseFunctionWebSearch):
                 additional_kwargs["built_in_tool_calls"].append(item)
             elif isinstance(item, ResponseComputerToolCall):
@@ -504,9 +517,6 @@ class OpenAIResponses(FunctionCallingLLM):
                     )
                 )
 
-        if tool_calls and message:
-            message.additional_kwargs["tool_calls"] = tool_calls
-
         return ChatResponse(message=message, additional_kwargs=additional_kwargs)
 
     @llm_retry_decorator
@@ -542,7 +552,6 @@ class OpenAIResponses(FunctionCallingLLM):
     @staticmethod
     def process_response_event(
         event: ResponseStreamEvent,
-        tool_calls: List[ResponseFunctionToolCall],
         built_in_tool_calls: List[Any],
         additional_kwargs: Dict[str, Any],
         current_tool_call: Optional[ResponseFunctionToolCall],
@@ -550,7 +559,6 @@ class OpenAIResponses(FunctionCallingLLM):
         previous_response_id: Optional[str] = None,
     ) -> Tuple[
         List[ContentBlock],
-        List[ResponseFunctionToolCall],
         List[Any],
         Dict[str, Any],
         Optional[ResponseFunctionToolCall],
@@ -591,6 +599,7 @@ class OpenAIResponses(FunctionCallingLLM):
         elif isinstance(event, ResponseTextDeltaEvent):
             # Text content is being added
             delta = event.delta
+            blocks.append(TextBlock(text=delta))
         elif isinstance(event, ResponseImageGenCallPartialImageEvent):
             # Partial image
             if event.partial_image_b64:
@@ -609,10 +618,12 @@ class OpenAIResponses(FunctionCallingLLM):
             if current_tool_call is not None:
                 current_tool_call.arguments = event.arguments
                 current_tool_call.status = "completed"
-
-                # append a copy of the tool call to the list
-                tool_calls.append(
-                    ResponseFunctionToolCall(**current_tool_call.model_dump())
+                blocks.append(
+                    ToolCallBlock(
+                        tool_name=current_tool_call.name,
+                        tool_kwargs=current_tool_call.arguments,
+                        tool_call_id=current_tool_call.call_id,
+                    )
                 )
 
                 # clear the current tool call
@@ -658,7 +669,6 @@ class OpenAIResponses(FunctionCallingLLM):
 
         return (
             blocks,
-            tool_calls,
             built_in_tool_calls,
             additional_kwargs,
             current_tool_call,
@@ -677,7 +687,6 @@ class OpenAIResponses(FunctionCallingLLM):
         )
 
         def gen() -> ChatResponseGen:
-            tool_calls = []
             built_in_tool_calls = []
             additional_kwargs = {"built_in_tool_calls": []}
             current_tool_call: Optional[ResponseFunctionToolCall] = None
@@ -691,7 +700,6 @@ class OpenAIResponses(FunctionCallingLLM):
                 # Process the event and update state
                 (
                     blocks,
-                    tool_calls,
                     built_in_tool_calls,
                     additional_kwargs,
                     current_tool_call,
@@ -699,7 +707,6 @@ class OpenAIResponses(FunctionCallingLLM):
                     delta,
                 ) = OpenAIResponses.process_response_event(
                     event=event,
-                    tool_calls=tool_calls,
                     built_in_tool_calls=built_in_tool_calls,
                     additional_kwargs=additional_kwargs,
                     current_tool_call=current_tool_call,
@@ -721,9 +728,6 @@ class OpenAIResponses(FunctionCallingLLM):
                     message=ChatMessage(
                         role=MessageRole.ASSISTANT,
                         blocks=blocks,
-                        additional_kwargs={"tool_calls": tool_calls}
-                        if tool_calls
-                        else {},
                     ),
                     delta=delta,
                     raw=event,
@@ -801,7 +805,6 @@ class OpenAIResponses(FunctionCallingLLM):
         )
 
         async def gen() -> ChatResponseAsyncGen:
-            tool_calls = []
             built_in_tool_calls = []
             additional_kwargs = {"built_in_tool_calls": []}
             current_tool_call: Optional[ResponseFunctionToolCall] = None
@@ -817,7 +820,6 @@ class OpenAIResponses(FunctionCallingLLM):
                 # Process the event and update state
                 (
                     blocks,
-                    tool_calls,
                     built_in_tool_calls,
                     additional_kwargs,
                     current_tool_call,
@@ -825,7 +827,6 @@ class OpenAIResponses(FunctionCallingLLM):
                     delta,
                 ) = OpenAIResponses.process_response_event(
                     event=event,
-                    tool_calls=tool_calls,
                     built_in_tool_calls=built_in_tool_calls,
                     additional_kwargs=additional_kwargs,
                     current_tool_call=current_tool_call,
@@ -847,9 +848,6 @@ class OpenAIResponses(FunctionCallingLLM):
                     message=ChatMessage(
                         role=MessageRole.ASSISTANT,
                         blocks=blocks,
-                        additional_kwargs={"tool_calls": tool_calls}
-                        if tool_calls
-                        else {},
                     ),
                     delta=delta,
                     raw=event,
@@ -915,9 +913,11 @@ class OpenAIResponses(FunctionCallingLLM):
         **kwargs: Any,
     ) -> List[ToolSelection]:
         """Predict and call the tool."""
-        tool_calls: List[ResponseFunctionToolCall] = (
-            response.message.additional_kwargs.get("tool_calls", [])
-        )
+        tool_calls: List[ToolCallBlock] = [
+            block
+            for block in response.message.blocks
+            if isinstance(block, ToolCallBlock)
+        ]
 
         if len(tool_calls) < 1:
             if error_on_no_tool_call:
@@ -931,14 +931,14 @@ class OpenAIResponses(FunctionCallingLLM):
         for tool_call in tool_calls:
             # this should handle both complete and partial jsons
             try:
-                argument_dict = parse_partial_json(tool_call.arguments)
-            except ValueError:
+                argument_dict = parse_partial_json(cast(str, tool_call.tool_kwargs))
+            except Exception:
                 argument_dict = {}
 
             tool_selections.append(
                 ToolSelection(
-                    tool_id=tool_call.call_id,
-                    tool_name=tool_call.name,
+                    tool_id=tool_call.tool_call_id or "",
+                    tool_name=tool_call.tool_name,
                     tool_kwargs=argument_dict,
                 )
             )

{llama_index_llms_openai-0.6.3 → llama_index_llms_openai-0.6.5}/llama_index/llms/openai/utils.py

@@ -30,6 +30,8 @@ from llama_index.core.base.llms.types import (
     AudioBlock,
     DocumentBlock,
     ThinkingBlock,
+    ToolCallBlock,
+    ContentBlock,
 )
 from llama_index.core.bridge.pydantic import BaseModel
 
@@ -199,9 +201,9 @@ JSON_SCHEMA_MODELS = [
     "o1-pro",
     "o3",
     "o3-mini",
-    "gpt-4.1",
     "gpt-4o",
     "gpt-4.1",
+    "gpt-5",
 ]
 
 
@@ -398,6 +400,30 @@ def to_openai_message_dict(
                     },
                 }
             )
+        elif isinstance(block, ToolCallBlock):
+            try:
+                function_dict = {
+                    "type": "function",
+                    "function": {
+                        "name": block.tool_name,
+                        "arguments": block.tool_kwargs,
+                    },
+                    "id": block.tool_call_id,
+                }
+
+                if len(content) == 0 or content[-1]["type"] != "text":
+                    content.append(
+                        {"type": "text", "text": "", "tool_calls": [function_dict]}
+                    )
+                elif content[-1]["type"] == "text" and "tool_calls" in content[-1]:
+                    content[-1]["tool_calls"].append(function_dict)
+                elif content[-1]["type"] == "text" and "tool_calls" not in content[-1]:
+                    content[-1]["tool_calls"] = [function_dict]
+            except Exception:
+                logger.warning(
+                    f"It was not possible to convert ToolCallBlock with call id {block.tool_call_id or '`no call id`'} to a valid message, skipping..."
+                )
+                continue
         else:
             msg = f"Unsupported content block type: {type(block).__name__}"
             raise ValueError(msg)
@@ -405,6 +431,9 @@ def to_openai_message_dict(
     # NOTE: Sending a null value (None) for Tool Message to OpenAI will cause error
     # It's only Allowed to send None if it's an Assistant Message and either a function call or tool calls were performed
     # Reference: https://platform.openai.com/docs/api-reference/chat/create
+    already_has_tool_calls = any(
+        isinstance(block, ToolCallBlock) for block in message.blocks
+    )
     content_txt = (
         None
         if content_txt == ""
@@ -412,6 +441,7 @@ def to_openai_message_dict(
         and (
             "function_call" in message.additional_kwargs
             or "tool_calls" in message.additional_kwargs
+            or already_has_tool_calls
         )
         else content_txt
     )
@@ -437,6 +467,13 @@ def to_openai_message_dict(
                 else content
             ),
         }
+        if already_has_tool_calls:
+            existing_tool_calls = []
+            for c in content:
+                existing_tool_calls.extend(c.get("tool_calls", []))
+
+            if existing_tool_calls:
+                message_dict["tool_calls"] = existing_tool_calls
 
     # TODO: O1 models do not support system prompts
     if (
@@ -447,10 +484,14 @@ def to_openai_message_dict(
         if message_dict["role"] == "system":
             message_dict["role"] = "developer"
 
-    # NOTE: openai messages have additional arguments:
-    # - function messages have `name`
-    # - assistant messages have optional `function_call`
-    message_dict.update(message.additional_kwargs)
+    if (
+        "tool_calls" in message.additional_kwargs
+        or "function_call" in message.additional_kwargs
+    ) and not already_has_tool_calls:
+        message_dict.update(message.additional_kwargs)
+
+    if "tool_call_id" in message.additional_kwargs:
+        message_dict["tool_call_id"] = message.additional_kwargs["tool_call_id"]
 
     null_keys = [key for key, value in message_dict.items() if value is None]
     # if drop_none is True, remove keys with None values
@@ -469,6 +510,8 @@ def to_openai_responses_message_dict(
     """Convert a ChatMessage to an OpenAI message dict."""
     content = []
     content_txt = ""
+    tool_calls = []
+    reasoning = []
 
     for block in message.blocks:
         if isinstance(block, TextBlock):
@@ -512,13 +555,41 @@ def to_openai_responses_message_dict(
                     }
                 )
         elif isinstance(block, ThinkingBlock):
-            if block.content:
-                content.append({"type": "output_text", "text": block.content})
-                content_txt += block.content
+            if block.content and "id" in block.additional_information:
+                reasoning.append(
+                    {
+                        "type": "reasoning",
+                        "id": block.additional_information["id"],
+                        "summary": [
+                            {"type": "summary_text", "text": block.content or ""}
+                        ],
+                    }
+                )
+        elif isinstance(block, ToolCallBlock):
+            tool_calls.extend(
+                [
+                    {
+                        "type": "function_call",
+                        "arguments": block.tool_kwargs,
+                        "call_id": block.tool_call_id,
+                        "name": block.tool_name,
+                    }
+                ]
+            )
         else:
             msg = f"Unsupported content block type: {type(block).__name__}"
             raise ValueError(msg)
 
+    if "tool_calls" in message.additional_kwargs:
+        message_dicts = [
+            tool_call if isinstance(tool_call, dict) else tool_call.model_dump()
+            for tool_call in message.additional_kwargs["tool_calls"]
+        ]
+
+        return [*reasoning, *message_dicts]
+    elif tool_calls:
+        return [*reasoning, *tool_calls]
+
     # NOTE: Sending a null value (None) for Tool Message to OpenAI will cause error
     # It's only Allowed to send None if it's an Assistant Message and either a function call or tool calls were performed
     # Reference: https://platform.openai.com/docs/api-reference/chat/create
@@ -553,13 +624,6 @@ def to_openai_responses_message_dict(
         }
 
         return message_dict
-    elif "tool_calls" in message.additional_kwargs:
-        message_dicts = [
-            tool_call if isinstance(tool_call, dict) else tool_call.model_dump()
-            for tool_call in message.additional_kwargs["tool_calls"]
-        ]
-
-        return message_dicts
 
     # there are some cases (like image generation or MCP tool call) that only support the string input
     # this is why, if context_txt is a non-empty string, all the blocks are TextBlocks and the role is user, we return directly context_txt
@@ -596,6 +660,9 @@ def to_openai_responses_message_dict(
         for key in null_keys:
             message_dict.pop(key)
 
+    if reasoning:
+        return [*reasoning, message_dict]
+
     return message_dict  # type: ignore
 
 
@@ -648,13 +715,22 @@ def from_openai_message(
     role = openai_message.role
     # NOTE: Azure OpenAI returns function calling messages without a content key
     if "text" in modalities and openai_message.content:
-        blocks = [TextBlock(text=openai_message.content or "")]
+        blocks: List[ContentBlock] = [TextBlock(text=openai_message.content or "")]
     else:
-        blocks = []
+        blocks: List[ContentBlock] = []
 
     additional_kwargs: Dict[str, Any] = {}
     if openai_message.tool_calls:
         tool_calls: List[ChatCompletionMessageToolCall] = openai_message.tool_calls
+        for tool_call in tool_calls:
+            if tool_call.function:
+                blocks.append(
+                    ToolCallBlock(
+                        tool_call_id=tool_call.id,
+                        tool_name=tool_call.function.name or "",
+                        tool_kwargs=tool_call.function.arguments or {},
+                    )
+                )
         additional_kwargs.update(tool_calls=tool_calls)
 
     if openai_message.audio and "audio" in modalities:
@@ -742,6 +818,14 @@ def from_openai_message_dict(message_dict: dict) -> ChatMessage:
                     blocks.append(ImageBlock(image=img, detail=detail))
                 else:
                     blocks.append(ImageBlock(url=img, detail=detail))
+            elif t == "function_call":
+                blocks.append(
+                    ToolCallBlock(
+                        tool_call_id=elem.get("call_id"),
+                        tool_name=elem.get("name", ""),
+                        tool_kwargs=elem.get("arguments", {}),
+                    )
+                )
             else:
                 msg = f"Unsupported message type: {t}"
                 raise ValueError(msg)

{llama_index_llms_openai-0.6.3 → llama_index_llms_openai-0.6.5}/pyproject.toml

@@ -27,13 +27,13 @@ dev = [
 
 [project]
 name = "llama-index-llms-openai"
-version = "0.6.3"
+version = "0.6.5"
 description = "llama-index llms openai integration"
 authors = [{name = "llama-index"}]
 requires-python = ">=3.9,<4.0"
 readme = "README.md"
 license = "MIT"
-dependencies = ["openai>=1.108.1,<2", "llama-index-core>=0.14.3,<0.15"]
+dependencies = ["openai>=1.108.1,<2", "llama-index-core>=0.14.5,<0.15"]
 
 [tool.codespell]
 check-filenames = true