llama-index-llms-openai 0.3.1__py3-none-any.whl → 0.3.3__py3-none-any.whl

llama_index/llms/openai/base.py

@@ -2,10 +2,10 @@ import functools
 from typing import (
     TYPE_CHECKING,
     Any,
-    Generator,
     Awaitable,
     Callable,
     Dict,
+    Generator,
     List,
     Optional,
     Protocol,
@@ -18,6 +18,8 @@ from typing import (
 
 import httpx
 import tiktoken
+
+import llama_index.core.instrumentation as instrument
 from llama_index.core.base.llms.generic_utils import (
     achat_to_completion_decorator,
     acompletion_to_chat_decorator,
@@ -39,7 +41,11 @@ from llama_index.core.base.llms.types import (
     LLMMetadata,
     MessageRole,
 )
-from llama_index.core.bridge.pydantic import Field, PrivateAttr
+from llama_index.core.bridge.pydantic import (
+    BaseModel,
+    Field,
+    PrivateAttr,
+)
 from llama_index.core.callbacks import CallbackManager
 from llama_index.core.constants import (
     DEFAULT_TEMPERATURE,
@@ -50,7 +56,8 @@ from llama_index.core.llms.callbacks import (
 )
 from llama_index.core.llms.function_calling import FunctionCallingLLM
 from llama_index.core.llms.llm import ToolSelection
-from llama_index.core.types import BaseOutputParser, PydanticProgramMode, Model
+from llama_index.core.llms.utils import parse_partial_json
+from llama_index.core.types import BaseOutputParser, Model, PydanticProgramMode
 from llama_index.llms.openai.utils import (
     O1_MODELS,
     OpenAIToolCall,
@@ -62,14 +69,10 @@ from llama_index.llms.openai.utils import (
     is_function_calling_model,
     openai_modelname_to_contextsize,
     resolve_openai_credentials,
-    to_openai_message_dicts,
     resolve_tool_choice,
+    to_openai_message_dicts,
     update_tool_calls,
 )
-from llama_index.core.bridge.pydantic import (
-    BaseModel,
-)
-
 from openai import AsyncOpenAI, AzureOpenAI
 from openai import OpenAI as SyncOpenAI
 from openai.types.chat.chat_completion_chunk import (
@@ -77,9 +80,6 @@ from openai.types.chat.chat_completion_chunk import (
     ChoiceDelta,
     ChoiceDeltaToolCall,
 )
-from llama_index.core.llms.utils import parse_partial_json
-
-import llama_index.core.instrumentation as instrument
 
 dispatcher = instrument.get_dispatcher(__name__)
 
@@ -89,7 +89,7 @@ if TYPE_CHECKING:
 DEFAULT_OPENAI_MODEL = "gpt-3.5-turbo"
 
 
-def llm_retry_decorator(f: Callable[[Any], Any]) -> Callable[[Any], Any]:
+def llm_retry_decorator(f: Callable[..., Any]) -> Callable[..., Any]:
     @functools.wraps(f)
     def wrapper(self, *args: Any, **kwargs: Any) -> Any:
         max_retries = getattr(self, "max_retries", 0)
@@ -112,7 +112,7 @@ def llm_retry_decorator(f: Callable[[Any], Any]) -> Callable[[Any], Any]:
 class Tokenizer(Protocol):
     """Tokenizers support an encode function that returns a list of ints."""
 
-    def encode(self, text: str) -> List[int]:
+    def encode(self, text: str) -> List[int]:  # fmt: skip
         ...
 
 
@@ -828,7 +828,7 @@ class OpenAI(FunctionCallingLLM):
 
     def _prepare_chat_with_tools(
         self,
-        tools: List["BaseTool"],
+        tools: Sequence["BaseTool"],
         user_msg: Optional[Union[str, ChatMessage]] = None,
         chat_history: Optional[List[ChatMessage]] = None,
         verbose: bool = False,
@@ -850,9 +850,8 @@ class OpenAI(FunctionCallingLLM):
             for tool_spec in tool_specs:
                 if tool_spec["type"] == "function":
                     tool_spec["function"]["strict"] = strict
-                    tool_spec["function"]["parameters"][
-                        "additionalProperties"
-                    ] = False  # in current openai 1.40.0 it is always false.
+                    # in current openai 1.40.0 it is always false.
+                    tool_spec["function"]["parameters"]["additionalProperties"] = False
 
         if isinstance(user_msg, str):
             user_msg = ChatMessage(role=MessageRole.USER, content=user_msg)
@@ -871,7 +870,7 @@ class OpenAI(FunctionCallingLLM):
     def _validate_chat_with_tools_response(
         self,
         response: ChatResponse,
-        tools: List["BaseTool"],
+        tools: Sequence["BaseTool"],
         allow_parallel_tool_calls: bool = False,
         **kwargs: Any,
     ) -> ChatResponse:
@@ -926,8 +925,10 @@ class OpenAI(FunctionCallingLLM):
     ) -> BaseModel:
         """Structured predict."""
         llm_kwargs = llm_kwargs or {}
+        all_kwargs = {**llm_kwargs, **kwargs}
+
         llm_kwargs["tool_choice"] = (
-            "required" if "tool_choice" not in llm_kwargs else llm_kwargs["tool_choice"]
+            "required" if "tool_choice" not in all_kwargs else all_kwargs["tool_choice"]
         )
         # by default structured prediction uses function calling to extract structured outputs
         # here we force tool_choice to be required
@@ -939,8 +940,10 @@ class OpenAI(FunctionCallingLLM):
     ) -> BaseModel:
         """Structured predict."""
         llm_kwargs = llm_kwargs or {}
+        all_kwargs = {**llm_kwargs, **kwargs}
+
         llm_kwargs["tool_choice"] = (
-            "required" if "tool_choice" not in llm_kwargs else llm_kwargs["tool_choice"]
+            "required" if "tool_choice" not in all_kwargs else all_kwargs["tool_choice"]
         )
         # by default structured prediction uses function calling to extract structured outputs
         # here we force tool_choice to be required
@@ -952,22 +955,28 @@ class OpenAI(FunctionCallingLLM):
     ) -> Generator[Union[Model, List[Model]], None, None]:
         """Stream structured predict."""
         llm_kwargs = llm_kwargs or {}
+        all_kwargs = {**llm_kwargs, **kwargs}
+
         llm_kwargs["tool_choice"] = (
-            "required" if "tool_choice" not in llm_kwargs else llm_kwargs["tool_choice"]
+            "required" if "tool_choice" not in all_kwargs else all_kwargs["tool_choice"]
         )
         # by default structured prediction uses function calling to extract structured outputs
         # here we force tool_choice to be required
         return super().stream_structured_predict(*args, llm_kwargs=llm_kwargs, **kwargs)
 
     @dispatcher.span
-    def stream_structured_predict(
+    async def astream_structured_predict(
         self, *args: Any, llm_kwargs: Optional[Dict[str, Any]] = None, **kwargs: Any
     ) -> Generator[Union[Model, List[Model]], None, None]:
         """Stream structured predict."""
         llm_kwargs = llm_kwargs or {}
+        all_kwargs = {**llm_kwargs, **kwargs}
+
         llm_kwargs["tool_choice"] = (
-            "required" if "tool_choice" not in llm_kwargs else llm_kwargs["tool_choice"]
+            "required" if "tool_choice" not in all_kwargs else all_kwargs["tool_choice"]
         )
         # by default structured prediction uses function calling to extract structured outputs
         # here we force tool_choice to be required
-        return super().stream_structured_predict(*args, llm_kwargs=llm_kwargs, **kwargs)
+        return await super().astream_structured_predict(
+            *args, llm_kwargs=llm_kwargs, **kwargs
+        )

llama_index/llms/openai/utils.py

@@ -3,9 +3,6 @@ import os
 from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple, Type, Union
 
 from deprecated import deprecated
-from llama_index.core.base.llms.types import ChatMessage, LogProb, CompletionResponse
-from llama_index.core.bridge.pydantic import BaseModel
-from llama_index.core.base.llms.generic_utils import get_from_param_or_env
 from tenacity import (
     before_sleep_log,
     retry,
@@ -18,12 +15,19 @@ from tenacity import (
 from tenacity.stop import stop_base
 
 import openai
+from llama_index.core.base.llms.generic_utils import get_from_param_or_env
+from llama_index.core.base.llms.types import (
+    ChatMessage,
+    ImageBlock,
+    LogProb,
+    TextBlock,
+)
+from llama_index.core.bridge.pydantic import BaseModel
 from openai.types.chat import ChatCompletionMessageParam, ChatCompletionMessageToolCall
 from openai.types.chat.chat_completion_chunk import ChoiceDeltaToolCall
 from openai.types.chat.chat_completion_message import ChatCompletionMessage
 from openai.types.chat.chat_completion_token_logprob import ChatCompletionTokenLogprob
 from openai.types.completion_choice import Logprobs
-from openai.types.completion import Completion
 
 DEFAULT_OPENAI_API_TYPE = "open_ai"
 DEFAULT_OPENAI_API_BASE = "https://api.openai.com/v1"
@@ -190,7 +194,8 @@ def create_retry_decorator(
 
 
 def openai_modelname_to_contextsize(modelname: str) -> int:
-    """Calculate the maximum number of tokens possible to generate for a model.
+    """
+    Calculate the maximum number of tokens possible to generate for a model.
 
     Args:
         modelname: The modelname we want to know the context size for.
@@ -250,10 +255,35 @@ def is_function_calling_model(model: str) -> bool:
 def to_openai_message_dict(
     message: ChatMessage, drop_none: bool = False, model: Optional[str] = None
 ) -> ChatCompletionMessageParam:
-    """Convert generic message to OpenAI message dict."""
+    """Convert a ChatMessage to an OpenAI message dict."""
+    content = []
+    for block in message.blocks:
+        if isinstance(block, TextBlock):
+            content.append({"type": "text", "text": block.text})
+        elif isinstance(block, ImageBlock):
+            if block.url:
+                content.append(
+                    {"type": "image_url", "image_url": {"url": str(block.url)}}
+                )
+            else:
+                img_bytes = block.resolve_image(as_base64=True).read()
+                img_str = img_bytes.decode("utf-8")
+                content.append(
+                    {
+                        "type": "image_url",
+                        "image_url": {
+                            "url": f"data:{block.image_mimetype};base64,{img_str}",
+                            "detail": block.detail or "low",
+                        },
+                    }
+                )
+        else:
+            msg = f"Unsupported content block type: {type(block).__name__}"
+            raise ValueError(msg)
+
     message_dict = {
         "role": message.role.value,
-        "content": message.content,
+        "content": content,
     }
 
     # TODO: O1 models do not support system prompts
@@ -314,7 +344,7 @@ def from_openai_token_logprob(
                 LogProb(token=el.token, logprob=el.logprob, bytes=el.bytes or [])
                 for el in openai_token_logprob.top_logprobs
             ]
-        except Exception as e:
+        except Exception:
             print(openai_token_logprob)
             raise
     return result
@@ -332,7 +362,7 @@ def from_openai_token_logprobs(
 
 
 def from_openai_completion_logprob(
-    openai_completion_logprob: Dict[str, float]
+    openai_completion_logprob: Dict[str, float],
 ) -> List[LogProb]:
     """Convert openai completion logprobs to generic list of LogProb."""
     return [
@@ -354,11 +384,6 @@ def from_openai_completion_logprobs(
     return result
 
 
-def from_openai_completion(openai_completion: Completion) -> CompletionResponse:
-    """Convert openai completion to CompletionResponse."""
-    text = openai_completion.choices[0].text
-
-
 def from_openai_messages(
     openai_messages: Sequence[ChatCompletionMessage],
 ) -> List[ChatMessage]:
@@ -370,13 +395,32 @@ def from_openai_message_dict(message_dict: dict) -> ChatMessage:
     """Convert openai message dict to generic message."""
     role = message_dict["role"]
     # NOTE: Azure OpenAI returns function calling messages without a content key
-    content = message_dict.get("content", None)
+    content = message_dict.get("content")
+    blocks = []
+    if isinstance(content, list):
+        for elem in content:
+            t = elem.get("type")
+            if t == "text":
+                blocks.append(TextBlock(text=elem.get("text")))
+            elif t == "image_url":
+                img = elem["image_url"]["url"]
+                detail = elem["image_url"]["detail"]
+                if img.startswith("data:"):
+                    blocks.append(ImageBlock(image=img, detail=detail))
+                else:
+                    blocks.append(ImageBlock(url=img, detail=detail))
+            else:
+                msg = f"Unsupported message type: {t}"
+                raise ValueError(msg)
+        content = None
 
     additional_kwargs = message_dict.copy()
     additional_kwargs.pop("role")
     additional_kwargs.pop("content", None)
 
-    return ChatMessage(role=role, content=content, additional_kwargs=additional_kwargs)
+    return ChatMessage(
+        role=role, content=content, additional_kwargs=additional_kwargs, blocks=blocks
+    )
 
 
 def from_openai_message_dicts(message_dicts: Sequence[dict]) -> List[ChatMessage]:
@@ -386,7 +430,8 @@ def from_openai_message_dicts(message_dicts: Sequence[dict]) -> List[ChatMessage
 
 @deprecated("Deprecated in favor of `to_openai_tool`, which should be used instead.")
 def to_openai_function(pydantic_class: Type[BaseModel]) -> Dict[str, Any]:
-    """Deprecated in favor of `to_openai_tool`.
+    """
+    Deprecated in favor of `to_openai_tool`.
 
     Convert pydantic class to OpenAI function.
     """
@@ -412,7 +457,8 @@ def resolve_openai_credentials(
     api_base: Optional[str] = None,
     api_version: Optional[str] = None,
 ) -> Tuple[Optional[str], str, str]:
-    """ "Resolve OpenAI credentials.
+    """
+    "Resolve OpenAI credentials.
 
     The order of precedence is:
     1. param
@@ -443,7 +489,8 @@ def validate_openai_api_key(api_key: Optional[str] = None) -> None:
 
 
 def resolve_tool_choice(tool_choice: Union[str, dict] = "auto") -> Union[str, dict]:
-    """Resolve tool choice.
+    """
+    Resolve tool choice.
 
     If tool_choice is a function name string, return the appropriate dict.
     """

{llama_index_llms_openai-0.3.1.dist-info → llama_index_llms_openai-0.3.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: llama-index-llms-openai
-Version: 0.3.1
+Version: 0.3.3
 Summary: llama-index llms openai integration
 License: MIT
 Author: llama-index
@@ -11,7 +11,7 @@ Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
-Requires-Dist: llama-index-core (>=0.12.0,<0.13.0)
+Requires-Dist: llama-index-core (>=0.12.4,<0.13.0)
 Requires-Dist: openai (>=1.40.0,<2.0.0)
 Description-Content-Type: text/markdown
 

llama_index_llms_openai-0.3.3.dist-info/RECORD

@@ -0,0 +1,6 @@
+llama_index/llms/openai/__init__.py,sha256=vm3cIBSGkBFlE77GyfyN0EhpJcnJZN95QMhPN53EkbE,148
+llama_index/llms/openai/base.py,sha256=V__JjM8FSTuvQdeb-kqJOzIX3d1v9x4bbOGDxs-O56s,35637
+llama_index/llms/openai/utils.py,sha256=bQecgCnmtsMJAncuSqVrBS3aLU59kY4_0f0kR_jDu8o,17654
+llama_index_llms_openai-0.3.3.dist-info/METADATA,sha256=-iwROPhZcCSZJl9SbQ15tVZB0JgkJFFKWTA_TnqGQ5Q,3320
+llama_index_llms_openai-0.3.3.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+llama_index_llms_openai-0.3.3.dist-info/RECORD,,

llama_index_llms_openai-0.3.1.dist-info/RECORD

@@ -1,6 +0,0 @@
-llama_index/llms/openai/__init__.py,sha256=vm3cIBSGkBFlE77GyfyN0EhpJcnJZN95QMhPN53EkbE,148
-llama_index/llms/openai/base.py,sha256=wg509NgBHVLgB1Z9bevMjJenu3BFf4WZvlGP40eVhj8,35460
-llama_index/llms/openai/utils.py,sha256=3fHw4U-fRa6yADE5DjFbJWs6gagSt3-aqUn8oHdm8Ec,16180
-llama_index_llms_openai-0.3.1.dist-info/METADATA,sha256=wwtRg7GXHqRsgJx2UmAUbUdZfVoO2rhTWX7vgrlhqGg,3320
-llama_index_llms_openai-0.3.1.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-llama_index_llms_openai-0.3.1.dist-info/RECORD,,