freeplay 0.3.19__py3-none-any.whl → 0.3.20__py3-none-any.whl

freeplay/resources/adapters.py

@@ -3,6 +3,7 @@ from dataclasses import dataclass
 from typing import Protocol, Dict, List, Union, Any
 
 from freeplay.errors import FreeplayConfigurationError
+from freeplay.support import MediaType
 
 
 @dataclass
@@ -11,12 +12,16 @@ class TextContent:
 
 
 @dataclass
-class ImageContentUrl:
+class MediaContentUrl:
+    slot_name: str
+    type: MediaType
     url: str
 
 
 @dataclass
-class ImageContentBase64:
+class MediaContentBase64:
+    slot_name: str
+    type: MediaType
     content_type: str
     data: str
 
@@ -58,24 +63,28 @@ class AnthropicAdapter(LLMAdapter):
         return anthropic_messages
 
     @staticmethod
-    def __map_content(content: Union[TextContent, ImageContentBase64, ImageContentUrl]) -> Dict[str, Any]:
+    def __map_content(content: Union[TextContent, MediaContentBase64, MediaContentUrl]) -> Dict[str, Any]:
         if isinstance(content, TextContent):
             return {
                 "type": "text",
                 "text": content.text
             }
-        elif isinstance(content, ImageContentBase64):
+        if content.type == "audio" or content.type == "video":
+            raise ValueError("Anthropic does not support audio or video content")
+
+        media_type = "image" if content.type == "image" else "document"
+        if isinstance(content, MediaContentBase64):
             return {
-                "type": "image",
+                "type": media_type,
                 "source": {
                     "type": "base64",
                     "media_type": content.content_type,
                     "data": content.data,
                 }
             }
-        elif isinstance(content, ImageContentUrl):
+        elif isinstance(content, MediaContentUrl):
             return {
-                "type": "image",
+                "type": media_type,
                 "source": {
                     "type": "url",
                     "url": content.url,
@@ -101,28 +110,52 @@ class OpenAIAdapter(LLMAdapter):
         return openai_messages
 
     @staticmethod
-    def __map_content(content: Union[TextContent, ImageContentBase64, ImageContentUrl]) -> Dict[str, Any]:
+    def __map_content(content: Union[TextContent, MediaContentBase64, MediaContentUrl]) -> Dict[str, Any]:
         if isinstance(content, TextContent):
             return {
                 "type": "text",
                 "text": content.text
             }
-        elif isinstance(content, ImageContentBase64):
+        elif isinstance(content, MediaContentBase64):
+            return OpenAIAdapter.__format_base64_content(content)
+        elif isinstance(content, MediaContentUrl):
+            if content.type != "image":
+                raise ValueError("Message contains a non-image URL, but OpenAI only supports image URLs.")
+
             return {
                 "type": "image_url",
                 "image_url": {
-                    "url": f"data:{content.content_type};base64,{content.data}"
+                    "url": content.url
+                }
+            }
+        else:
+            raise ValueError(f"Unexpected content type {type(content)}")
+
+    @staticmethod
+    def __format_base64_content(content: MediaContentBase64) -> Dict[str, Any]:
+        if content.type == "audio":
+            return {
+                "type": "input_audio",
+                "input_audio": {
+                    "data": content.data,
+                    "format": content.content_type.split("/")[-1].replace("mpeg", "mp3")
+                }
+            }
+        elif content.type == "file":
+            return {
+                "type": "file",
+                "file": {
+                    "filename": f"{content.slot_name}.{content.content_type.split('/')[-1]}",
+                    "file_data": f"data:{content.content_type};base64,{content.data}"
                 }
             }
-        elif isinstance(content, ImageContentUrl):
+        else:
             return {
                 "type": "image_url",
                 "image_url": {
-                    "url": content.url
+                    "url": f"data:{content.content_type};base64,{content.data}"
                 }
             }
-        else:
-            raise ValueError(f"Unexpected content type {type(content)}")
 
 
 class Llama3Adapter(LLMAdapter):
@@ -163,17 +196,17 @@ class GeminiAdapter(LLMAdapter):
         return gemini_messages
 
     @staticmethod
-    def __map_content(content: Union[TextContent, ImageContentBase64, ImageContentUrl]) -> Dict[str, Any]:
+    def __map_content(content: Union[TextContent, MediaContentBase64, MediaContentUrl]) -> Dict[str, Any]:
         if isinstance(content, TextContent):
             return {"text": content.text}
-        elif isinstance(content, ImageContentBase64):
+        elif isinstance(content, MediaContentBase64):
             return {
                 "inline_data": {
                     "data": content.data,
                     "mime_type": content.content_type,
                 }
             }
-        elif isinstance(content, ImageContentUrl):
+        elif isinstance(content, MediaContentUrl):
             raise ValueError("Message contains an image URL, but image URLs are not supported by Gemini")
         else:
             raise ValueError(f"Unexpected content type {type(content)}")

freeplay/resources/prompts.py

@@ -25,7 +25,7 @@ from freeplay.errors import (
 )
 from freeplay.llm_parameters import LLMParameters
 from freeplay.model import InputVariables
-from freeplay.resources.adapters import MissingFlavorError, adaptor_for_flavor, ImageContentBase64, ImageContentUrl, \
+from freeplay.resources.adapters import MissingFlavorError, adaptor_for_flavor, MediaContentBase64, MediaContentUrl, \
     TextContent
 from freeplay.support import (
     CallSupport,
@@ -212,18 +212,16 @@ MediaInputMap = Dict[str, MediaInput]
 
 
 def extract_media_content(media_inputs: MediaInputMap, media_slots: List[MediaSlot]) -> List[
-    Union[ImageContentBase64, ImageContentUrl]]:
-    media_content: List[Union[ImageContentBase64, ImageContentUrl]] = []
+    Union[MediaContentBase64, MediaContentUrl]]:
+    media_content: List[Union[MediaContentBase64, MediaContentUrl]] = []
     for slot in media_slots:
-        if slot.type != "image":
-            continue
         file = media_inputs.get(slot.placeholder_name, None)
         if file is None:
             continue
         if isinstance(file, MediaInputUrl):
-            media_content.append(ImageContentUrl(url=file.url))
+            media_content.append(MediaContentUrl(type=slot.type, url=file.url, slot_name=slot.placeholder_name))
         else:
-            media_content.append(ImageContentBase64(content_type=file.content_type, data=file.data))
+            media_content.append(MediaContentBase64(type=slot.type, content_type=file.content_type, data=file.data, slot_name=slot.placeholder_name))
 
     return media_content
 

freeplay/support.py

@@ -29,9 +29,12 @@ class ToolSchema:
 Role = Literal['system', 'user', 'assistant']
 
 
+MediaType = Literal["image", "audio", "video", "file"]
+
+
 @dataclass
 class MediaSlot:
-    type: Literal["image", "audio", "video", "file"]
+    type: MediaType
     placeholder_name: str
 
 

{freeplay-0.3.19.dist-info → freeplay-0.3.20.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: freeplay
-Version: 0.3.19
+Version: 0.3.20
 Summary: 
 License: MIT
 Author: FreePlay Engineering

{freeplay-0.3.19.dist-info → freeplay-0.3.20.dist-info}/RECORD

@@ -7,17 +7,17 @@ freeplay/llm_parameters.py,sha256=bQbfuC8EICF0XMZQa5pwI3FkQqxmCUVqHO3gYHy3Tg8,89
 freeplay/model.py,sha256=o0de_RZ2WTJ4m5OJw1ZVfC2xG6zBq_XShBrRt1laEjc,1405
 freeplay/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 freeplay/resources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-freeplay/resources/adapters.py,sha256=zq-cDTBgFziFHyAJX18rEM4Fv2NHrLg3PuuxPqmg6Vw,6902
+freeplay/resources/adapters.py,sha256=BFqzLoJhLJeXfUyPOKDnllN_3jlbbBH0z32tXv20R20,8188
 freeplay/resources/customer_feedback.py,sha256=bw8MfEOKbGgn4FOyvcADrcs9GhcpNXNTgxKjBjIzywE,899
-freeplay/resources/prompts.py,sha256=i1Lck6UTF0WhW5miK5CAIJl19BFoI760rSyjFdju6H4,22799
+freeplay/resources/prompts.py,sha256=poLt48UG_3q6qDMvMqk0x6Y4QM66Mt6LyASpfcG0jwo,22843
 freeplay/resources/recordings.py,sha256=z2ARII1jCnmNh1GU3hGnXZUz5IF_KhyayQum71k-h9c,9213
 freeplay/resources/sessions.py,sha256=J5A3CjiV2MFqQyxN3TWTvJaa9jmMza58mRFRq2v9iAk,3746
 freeplay/resources/test_cases.py,sha256=nXL_976RwSJDT6OWDM4GEzbcOzcGkJ9ulvb0XOzCRDM,2240
 freeplay/resources/test_runs.py,sha256=Tp2N-odInT5XEEWrEsVhdgfnsclOE8n92_C8gTwO2MI,2623
-freeplay/support.py,sha256=sj3JLD2syBxDyzWVWthWWwYWCUy4GzLNMSLSHegVvkk,12251
+freeplay/support.py,sha256=kQMItnMGZT5TOdPQsSiKOlBbBqO4AyR91vito6wt4JM,12275
 freeplay/utils.py,sha256=Xvt4mNLXLL7E6MI2hTuDLV5cl5Y83DgdjCZSyDGMjR0,3187
-freeplay-0.3.19.dist-info/LICENSE,sha256=_jzIw45hB1XHGxiQ8leZ0GH_X7bR_a8qgxaqnHbCUOo,1064
-freeplay-0.3.19.dist-info/METADATA,sha256=-RBwb7sFV4g4yNM1bREAO8mCF0cIFcWrnKmcPv9FJTI,1654
-freeplay-0.3.19.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
-freeplay-0.3.19.dist-info/entry_points.txt,sha256=32s3rf2UUCqiJT4jnClEXZhdXlvl30uwpcxz-Gsy4UU,54
-freeplay-0.3.19.dist-info/RECORD,,
+freeplay-0.3.20.dist-info/LICENSE,sha256=_jzIw45hB1XHGxiQ8leZ0GH_X7bR_a8qgxaqnHbCUOo,1064
+freeplay-0.3.20.dist-info/METADATA,sha256=3mkuwNZ5k_CGH3wtcIOCrI5lDPL2meBLE4_fqQW7vOc,1654
+freeplay-0.3.20.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
+freeplay-0.3.20.dist-info/entry_points.txt,sha256=32s3rf2UUCqiJT4jnClEXZhdXlvl30uwpcxz-Gsy4UU,54
+freeplay-0.3.20.dist-info/RECORD,,