mirascope 1.16.9__py3-none-any.whl → 1.17.0__py3-none-any.whl

mirascope/core/gemini/_utils/_message_param_converter.py

@@ -8,7 +8,14 @@ from google.generativeai.types import (
 )
 
 from mirascope.core import BaseMessageParam
-from mirascope.core.base import DocumentPart, ImagePart, TextPart
+from mirascope.core.base import (
+    AudioPart,
+    AudioURLPart,
+    DocumentPart,
+    ImagePart,
+    ImageURLPart,
+    TextPart,
+)
 from mirascope.core.base._utils._base_message_param_converter import (
     BaseMessageParamConverter,
 )
@@ -29,6 +36,28 @@ def _to_image_part(mime_type: str, data: bytes) -> ImagePart:
     return ImagePart(type="image", media_type=mime_type, image=data, detail=None)
 
 
+def _is_audio_mime(mime_type: str) -> bool:
+    return mime_type in [
+        "audio/wav",
+        "audio/mp3",
+        "audio/wav",
+        "audio/mp3",
+        "audio/aiff",
+        "audio/aac",
+        "audio/ogg",
+        "audio/flac",
+    ]
+
+
+def _to_audio_part(mime_type: str, data: bytes) -> AudioPart:
+    if not _is_audio_mime(mime_type):
+        raise ValueError(
+            f"Unsupported audio media type: {mime_type}. "
+            "Expected one of: audio/wav, audio/mp3, audio/aiff, audio/aac, audio/ogg, audio/flac."
+        )
+    return AudioPart(type="audio", media_type=mime_type, audio=data)
+
+
 def _to_document_part(mime_type: str, data: bytes) -> DocumentPart:
     if mime_type != "application/pdf":
         raise ValueError(
@@ -77,6 +106,8 @@ class GeminiMessageParamConverter(BaseMessageParamConverter):
                         data = blob.data
                         if _is_image_mime(mime):
                             content_list.append(_to_image_part(mime, data))
+                        elif _is_audio_mime(mime):
+                            content_list.append(_to_audio_part(mime, data))
                         elif mime == "application/pdf":
                             content_list.append(_to_document_part(mime, data))
                         else:
@@ -85,10 +116,25 @@ class GeminiMessageParamConverter(BaseMessageParamConverter):
                             )
 
                     elif part.file_data:
-                        # part.file_data.file_uri has Google storage URI like "gs://bucket_name/file_name"
-                        raise ValueError(
-                            f"FileData.file_uri is not support: {part.file_data}. Cannot convert to BaseMessageParam."
-                        )
+                        if _is_image_mime(part.file_data.mime_type):
+                            content_list.append(
+                                ImageURLPart(
+                                    type="image_url",
+                                    url=part.file_data.file_uri,
+                                    detail=None,
+                                )
+                            )
+                        elif _is_audio_mime(part.file_data.mime_type):
+                            content_list.append(
+                                AudioURLPart(
+                                    type="audio_url",
+                                    url=part.file_data.file_uri,
+                                )
+                            )
+                        else:
+                            raise ValueError(
+                                f"Unsupported file_data mime type: {part.file_data.mime_type}. Cannot convert to BaseMessageParam."
+                            )
                     elif part.function_call:
                         converted.append(
                             BaseMessageParam(

mirascope/core/gemini/_utils/_setup_call.py

@@ -11,7 +11,7 @@ from google.generativeai.types import (
     GenerateContentResponse,
     GenerationConfigDict,
 )
-from google.generativeai.types.content_types import ToolConfigDict
+from google.generativeai.types.content_types import ToolConfigDict, to_content
 from pydantic import BaseModel
 
 from ...base import BaseMessageParam, BaseTool, _utils
@@ -108,6 +108,7 @@ def setup_call(
     call_kwargs = cast(GeminiCallKwargs, base_call_kwargs)
     messages = cast(list[BaseMessageParam | ContentDict], messages)
     messages = convert_message_params(messages)
+
     if json_mode:
         generation_config = call_kwargs.get("generation_config", {})
         if is_dataclass(generation_config):
@@ -125,11 +126,20 @@ def setup_call(
             "allowed_function_names": [tool_types[0]._name()],
         }
         call_kwargs["tool_config"] = tool_config
-    call_kwargs |= {"contents": messages}
 
     if client is None:
         client = GenerativeModel(model_name=model)
 
+    if messages and messages[0]["role"] == "system":
+        system_instruction = client._system_instruction
+        system_instruction = (
+            list(system_instruction.parts) if system_instruction else []
+        )
+        system_instruction.extend(messages.pop(0)["parts"])  # pyright: ignore [reportArgumentType]
+        client._system_instruction = to_content(system_instruction)  # pyright: ignore [reportArgumentType]
+
+    call_kwargs |= {"contents": messages}
+
     create = (
         get_async_create_fn(client.generate_content_async)
         if fn_is_async(fn)

mirascope/core/groq/_utils/_convert_message_params.py

@@ -45,6 +45,15 @@ def convert_message_params(
                             },
                         }
                     )
+                elif part.type == "image_url":
+                    converted_content.append(
+                        {
+                            "type": "image_url",
+                            "image_url": {
+                                "url": part.url,
+                            },
+                        }
+                    )
                 elif part.type == "tool_call":
                     converted_message_param = {
                         "role": "assistant",

mirascope/core/groq/_utils/_message_param_converter.py

@@ -8,7 +8,7 @@ from mirascope.core.base import TextPart, ToolResultPart
 from mirascope.core.base._utils._base_message_param_converter import (
     BaseMessageParamConverter,
 )
-from mirascope.core.base.message_param import ToolCallPart
+from mirascope.core.base.message_param import ImageURLPart, ToolCallPart
 from mirascope.core.groq._utils import convert_message_params
 
 
@@ -70,7 +70,14 @@ class GroqMessageParamConverter(BaseMessageParamConverter):
                 for part in content:
                     if "text" in part:
                         contents.append(TextPart(type="text", text=part["text"]))
-                    # TODO: add support for image parts here
+                    elif "image_url" in part:
+                        contents.append(
+                            ImageURLPart(
+                                type="image_url",
+                                url=part["image_url"]["url"],
+                                detail=part["image_url"].get("detail"),
+                            )
+                        )
             if contents:
                 converted.append(
                     BaseMessageParam(role=message_param["role"], content=contents)

mirascope/core/mistral/_utils/_convert_message_params.py

@@ -69,6 +69,13 @@ def convert_message_params(
                             )
                         )
                     )
+                elif part.type == "image_url":
+                    converted_content.append(
+                        {
+                            "type": "image_url",
+                            "image_url": part.url,
+                        }
+                    )
                 elif part.type == "tool_call":
                     converted_message_params.append(
                         AssistantMessage(

mirascope/core/mistral/_utils/_message_param_converter.py

@@ -19,7 +19,22 @@ from mirascope.core.base._utils._base_message_param_converter import (
 from mirascope.core.mistral._utils import convert_message_params
 
 from ...base import BaseMessageParam, ImagePart, TextPart, ToolResultPart
-from ...base.message_param import ToolCallPart
+from ...base.message_param import ImageURLPart, ToolCallPart
+
+
+def _create_image_part_from_data_url(image_url: str) -> ImagePart | None:
+    match = re.match(r"data:(image/\w+);base64,(.+)", image_url)
+    if not match:
+        return None
+    mime_type = match.group(1)
+    image_base64 = match.group(2)
+    image_data = base64.b64decode(image_base64)
+    return ImagePart(
+        type="image",
+        media_type=mime_type,
+        image=image_data,
+        detail=None,
+    )
 
 
 class MistralMessageParamConverter(BaseMessageParamConverter):
@@ -102,44 +117,35 @@ class MistralMessageParamConverter(BaseMessageParamConverter):
                     elif isinstance(chunk, ImageURLChunk):
                         image_url = chunk.image_url
                         if isinstance(image_url, str):
-                            # Extract data from the data URL
-                            match = re.match(r"data:(image/\w+);base64,(.+)", image_url)
-                            if not match:
-                                raise ValueError(
-                                    "ImageURLChunk image_url is not in a supported data URL format."
-                                )
-                            mime_type = match.group(1)
-                            image_base64 = match.group(2)
-                            image_data = base64.b64decode(image_base64)
-                            converted_parts.append(
-                                ImagePart(
-                                    type="image",
-                                    media_type=mime_type,
-                                    image=image_data,
-                                    detail=None,
+                            if image_part := _create_image_part_from_data_url(
+                                image_url
+                            ):
+                                converted_parts.append(image_part)
+                            else:
+                                converted_parts.append(
+                                    ImageURLPart(
+                                        type="image_url",
+                                        url=image_url,
+                                        detail=None,
+                                    )
                                 )
-                            )
+
                         else:
                             img_url_str = image_url.url  # type: ignore
-                            match = re.match(
-                                r"data:(image/\w+);base64,(.+)", img_url_str
-                            )
-                            if not match:
-                                raise ValueError(
-                                    "ImageURLChunk image_url is not in a supported data URL format."
-                                )
-                            mime_type = match.group(1)
-                            image_base64 = match.group(2)
-                            image_data = base64.b64decode(image_base64)
-                            converted_parts.append(
-                                ImagePart(
-                                    type="image",
-                                    media_type=mime_type,
-                                    image=image_data,
-                                    detail=None,
+                            if image_part := _create_image_part_from_data_url(
+                                img_url_str
+                            ):
+                                converted_parts.append(image_part)
+                            else:
+                                converted_parts.append(
+                                    ImageURLPart(
+                                        type="image_url",
+                                        url=img_url_str,
+                                        detail=image_url.detail
+                                        if isinstance(image_url.detail, str)
+                                        else None,
+                                    )
                                 )
-                            )
-
                     elif isinstance(chunk, ReferenceChunk):
                         raise ValueError(
                             "ReferenceChunk is not supported for conversion to BaseMessageParam."

mirascope/core/openai/_utils/_convert_message_params.py

@@ -6,6 +6,8 @@ import json
 from openai.types.chat import ChatCompletionMessageParam
 
 from ...base import BaseMessageParam
+from ...base._utils import get_audio_type
+from ...base._utils._parse_content_template import _load_media
 
 
 def convert_message_params(
@@ -44,6 +46,16 @@ def convert_message_params(
                             },
                         }
                     )
+                elif part.type == "image_url":
+                    converted_content.append(
+                        {
+                            "type": "image_url",
+                            "image_url": {
+                                "url": part.url,
+                                "detail": part.detail if part.detail else "auto",
+                            },
+                        }
+                    )
                 elif part.type == "audio":
                     if part.media_type not in [
                         "audio/wav",
@@ -53,11 +65,36 @@ def convert_message_params(
                             f"Unsupported audio media type: {part.media_type}. "
                             "OpenAI currently only supports WAV and MP3 audio file types."
                         )
+                    data = (
+                        part.audio
+                        if isinstance(part.audio, str)
+                        else base64.b64encode(part.audio).decode("utf-8")
+                    )
                     converted_content.append(
                         {
                             "input_audio": {
                                 "format": part.media_type.split("/")[-1],
-                                "data": base64.b64encode(part.audio).decode("utf-8"),
+                                "data": data,
+                            },
+                            "type": "input_audio",
+                        }
+                    )
+                elif part.type == "audio_url":
+                    audio = _load_media(part.url)
+                    audio_type = get_audio_type(audio)
+                    if audio_type not in [
+                        "audio/wav",
+                        "audio/mp3",
+                    ]:
+                        raise ValueError(
+                            f"Unsupported audio media type: {audio_type}. "
+                            "OpenAI currently only supports WAV and MP3 audio file types."
+                        )
+                    converted_content.append(
+                        {
+                            "input_audio": {
+                                "format": audio_type.split("/")[-1],
+                                "data": base64.b64encode(audio).decode("utf-8"),
                             },
                             "type": "input_audio",
                         }

mirascope/core/openai/_utils/_message_param_converter.py

@@ -1,12 +1,19 @@
 """This module contains the OpenAIMessageParamConverter class."""
 
 import json
+from collections.abc import Iterable
 from typing import cast
 
 from openai.types.chat import ChatCompletionMessageParam
 
 from mirascope.core import BaseMessageParam
-from mirascope.core.base import TextPart, ToolCallPart, ToolResultPart
+from mirascope.core.base import (
+    AudioPart,
+    ImageURLPart,
+    TextPart,
+    ToolCallPart,
+    ToolResultPart,
+)
 from mirascope.core.base._utils._base_message_param_converter import (
     BaseMessageParamConverter,
 )
@@ -57,12 +64,29 @@ class OpenAIMessageParamConverter(BaseMessageParamConverter):
                     BaseMessageParam(role=message_param["role"], content=content)
                 )
                 continue
-            elif isinstance(content, list):
+            elif isinstance(content, Iterable):
                 for part in content:
-                    if "text" in part:
+                    if part["type"] == "text":
                         contents.append(TextPart(type="text", text=part["text"]))
+                    elif part["type"] == "image_url":
+                        image_url = part["image_url"]
+                        contents.append(
+                            ImageURLPart(
+                                type="image_url",
+                                url=image_url["url"],
+                                detail=image_url.get("detail", None),
+                            )
+                        )
+                    elif part["type"] == "input_audio":
+                        input_audio = part["input_audio"]
+                        contents.append(
+                            AudioPart(
+                                type="audio",
+                                media_type=f"audio/{input_audio['format']}",
+                                audio=input_audio["data"],
+                            )
+                        )
                     else:
-                        # TODO: add support for image and audio parts here
                         raise ValueError(part["refusal"])  # pyright: ignore [reportGeneralTypeIssues]
             if tool_calls := message_param.get("tool_calls"):
                 for tool_call in tool_calls:

mirascope/core/vertex/_utils/_convert_message_params.py

@@ -1,10 +1,16 @@
 """Utility for converting `BaseMessageParam` to `Content`"""
 
+import base64
+import io
+
+import PIL.Image
 from google.cloud.aiplatform_v1beta1.types import content as gapic_content_types
 from google.cloud.aiplatform_v1beta1.types import tool as gapic_tool_types
 from vertexai.generative_models import Content, Image, Part
 
 from ...base import BaseMessageParam
+from ...base._utils import get_audio_type
+from ...base._utils._parse_content_template import _load_media
 
 
 def convert_message_params(
@@ -22,15 +28,11 @@ def convert_message_params(
                 )  # pragma: no cover
             converted_message_params += [
                 Content(
-                    role="user",
+                    role="system",
                     parts=[
                         Part.from_text(content) if isinstance(content, str) else content
                     ],
                 ),
-                Content(
-                    role="model",
-                    parts=[Part.from_text("Ok! I will adhere to this system message.")],
-                ),
             ]
         elif isinstance((content := message_param.content), str):
             converted_message_params.append(
@@ -56,6 +58,29 @@ def convert_message_params(
                         )
                     image = Image.from_bytes(part.image)
                     converted_content.append(Part.from_image(image))
+                elif part.type == "image_url":
+                    # Should download the image to determine the media type
+                    image = PIL.Image.open(io.BytesIO(_load_media(part.url)))
+                    media_type = (
+                        PIL.Image.MIME[image.format]
+                        if image.format
+                        else "image/unknown"
+                    )
+                    if media_type not in [
+                        "image/jpeg",
+                        "image/png",
+                        "image/webp",
+                        "image/heic",
+                        "image/heif",
+                    ]:
+                        raise ValueError(
+                            f"Unsupported image media type: {media_type}. "
+                            "Gemini currently only supports JPEG, PNG, WebP, HEIC, "
+                            "and HEIF images."
+                        )
+                    converted_content.append(
+                        Part.from_uri(part.url, mime_type=media_type)
+                    )
                 elif part.type == "audio":
                     if part.media_type not in [
                         "audio/wav",
@@ -71,7 +96,32 @@ def convert_message_params(
                             "and FLAC audio file types."
                         )
                     converted_content.append(
-                        Part.from_data(mime_type=part.media_type, data=part.audio)
+                        Part.from_data(
+                            mime_type=part.media_type,
+                            data=part.audio
+                            if isinstance(part.audio, bytes)
+                            else base64.b64decode(part.audio),
+                        )
+                    )
+                elif part.type == "audio_url":
+                    # Should download the audio to determine the media type
+                    audio = _load_media(part.url)
+                    audio_type = get_audio_type(audio)
+                    if audio_type not in [
+                        "audio/wav",
+                        "audio/mp3",
+                        "audio/aiff",
+                        "audio/aac",
+                        "audio/ogg",
+                        "audio/flac",
+                    ]:
+                        raise ValueError(
+                            f"Unsupported audio media type: {audio_type}. "
+                            "Gemini currently only supports WAV, MP3, AIFF, AAC, OGG, "
+                            "and FLAC audio file types."
+                        )
+                    converted_content.append(
+                        Part.from_uri(part.url, mime_type=audio_type)
                     )
                 elif part.type == "tool_call":
                     if converted_content:

mirascope/core/vertex/_utils/_message_param_converter.py

@@ -7,7 +7,7 @@ from mirascope.core.base import DocumentPart, ImagePart, TextPart
 from mirascope.core.base._utils._base_message_param_converter import (
     BaseMessageParamConverter,
 )
-from mirascope.core.base.message_param import ToolCallPart, ToolResultPart
+from mirascope.core.base.message_param import ImageURLPart, ToolCallPart, ToolResultPart
 from mirascope.core.vertex._utils import convert_message_params
 
 
@@ -87,10 +87,18 @@ class VertexMessageParamConverter(BaseMessageParamConverter):
                         )
 
                 elif part.file_data:
-                    # part.file_data.file_uri has Google storage URI like "gs://bucket_name/file_name"
-                    raise ValueError(
-                        f"FileData.file_uri is not support: {part.file_data}. Cannot convert to BaseMessageParam."
-                    )
+                    if _is_image_mime(part.file_data.mime_type):
+                        contents.append(
+                            ImageURLPart(
+                                type="image_url",
+                                url=part.file_data.file_uri,
+                                detail=None,
+                            )
+                        )
+                    else:
+                        raise ValueError(
+                            f"FileData.file_uri is not support: {part.file_data}. Cannot convert to BaseMessageParam."
+                        )
                 elif part.function_call:
                     converted.append(
                         BaseMessageParam(

mirascope/core/vertex/_utils/_setup_call.py

@@ -108,6 +108,7 @@ def setup_call(
     call_kwargs = cast(VertexCallKwargs, base_call_kwargs)
     messages = cast(list[BaseMessageParam | Content], messages)
     messages = convert_message_params(messages)
+
     if json_mode:
         generation_config = call_kwargs.get(
             "generation_config",
@@ -131,11 +132,19 @@ def setup_call(
             )
         )
         call_kwargs["tool_config"] = tool_config
-    call_kwargs |= {"contents": messages}
 
     if client is None:
         client = GenerativeModel(model_name=model)
 
+    if messages and messages[0].role == "system":
+        system_instruction = client._system_instruction
+        if not isinstance(system_instruction, list):
+            system_instruction = [system_instruction] if system_instruction else []
+        system_instruction.extend(messages.pop(0).parts)
+        client._system_instruction = system_instruction
+
+    call_kwargs |= {"contents": messages}
+
     create = (
         cast(
             AsyncCreateFn[GenerationResponse, AsyncIterable[GenerationResponse]],

mirascope/llm/call_response.py

@@ -51,7 +51,11 @@ class CallResponse(
         response: BaseCallResponse[_ResponseT, _BaseToolT, Any, Any, Any, Any, Any],
     ) -> None:
         super().__init__(
-            **{field: getattr(response, field) for field in response.model_fields}
+            **{
+                field: getattr(response, field)
+                for field in response.model_fields
+                if field != "user_message_param"
+            }
         )
         object.__setattr__(self, "_response", response)
         object.__setattr__(

mirascope/retries/__init__.py

@@ -2,5 +2,10 @@
 
 from contextlib import suppress
 
+from .fallback import FallbackError, fallback
+
 with suppress(ImportError):
     from . import tenacity as tenacity
+
+
+__all__ = ["fallback", "FallbackError", "tenacity"]