videosdk-plugins-openai 0.0.19__py3-none-any.whl → 0.0.21__py3-none-any.whl

videosdk/plugins/openai/llm.py

@@ -1,12 +1,25 @@
 from __future__ import annotations
 
 import os
-from typing import Any, AsyncIterator
+from typing import Any, AsyncIterator, List, Union
 import json
 
 import httpx
 import openai
-from videosdk.agents import LLM, LLMResponse, ChatContext, ChatRole, ChatMessage, FunctionCall, FunctionCallOutput, ToolChoice, FunctionTool, is_function_tool, build_openai_schema
+from videosdk.agents import (
+    LLM,
+    LLMResponse,
+    ChatContext,
+    ChatRole,
+    ChatMessage,
+    FunctionCall,
+    FunctionCallOutput,
+    ToolChoice,
+    FunctionTool,
+    is_function_tool,
+    build_openai_schema,
+)
+from videosdk.agents.llm.chat_context import ChatContent, ImageContent
 
 class OpenAILLM(LLM):
     
@@ -62,35 +75,55 @@ class OpenAILLM(LLM):
         Yields:
             LLMResponse objects containing the model's responses
         """
+        def _format_content(content: Union[str, List[ChatContent]]):
+            if isinstance(content, str):
+                return content
+
+            formatted_parts = []
+            for part in content:
+                if isinstance(part, str):
+                    formatted_parts.append({"type": "text", "text": part})
+                elif isinstance(part, ImageContent):
+                    image_url_data = {"url": part.to_data_url()}
+                    if part.inference_detail != "auto":
+                        image_url_data["detail"] = part.inference_detail
+                    formatted_parts.append(
+                        {
+                            "type": "image_url",
+                            "image_url": image_url_data,
+                        }
+                    )
+            return formatted_parts
+
         completion_params = {
             "model": self.model,
             "messages": [
                 {
                     "role": msg.role.value,
-                    "content": msg.content,
-                    **({"name": msg.name} if hasattr(msg, 'name') else {})
-                } if isinstance(msg, ChatMessage) else
-                {
+                    "content": _format_content(msg.content),
+                    **({"name": msg.name} if hasattr(msg, "name") else {}),
+                }
+                if isinstance(msg, ChatMessage)
+                else {
                     "role": "assistant",
                     "content": None,
-                    "function_call": {
-                        "name": msg.name,
-                        "arguments": msg.arguments
-                    }
-                } if isinstance(msg, FunctionCall) else
-                {
+                    "function_call": {"name": msg.name, "arguments": msg.arguments},
+                }
+                if isinstance(msg, FunctionCall)
+                else {
                     "role": "function",
                     "name": msg.name,
-                    "content": msg.output
-                } if isinstance(msg, FunctionCallOutput) else None
+                    "content": msg.output,
+                }
+                if isinstance(msg, FunctionCallOutput)
+                else None
                 for msg in messages.items
-                if msg is not None 
+                if msg is not None
             ],
             "temperature": self.temperature,
             "stream": True,
             "max_tokens": self.max_completion_tokens,
         }
-
         if tools:
             formatted_tools = []
             for tool in tools:

videosdk/plugins/openai/tts.py

@@ -4,6 +4,7 @@ from typing import Any, AsyncIterator, Literal, Optional, Union
 import httpx
 import os
 import openai
+import asyncio
 
 from videosdk.agents import TTS
 
@@ -81,6 +82,7 @@ class OpenAITTS(TTS):
                 self.emit("error", "Audio track or event loop not set")
                 return
 
+            audio_data = b""
             async with self._client.audio.speech.with_streaming_response.create(
                 model=self.model,
                 voice=voice_id or self.voice,
@@ -91,13 +93,32 @@ class OpenAITTS(TTS):
             ) as response:
                 async for chunk in response.iter_bytes():
                     if chunk:
-                        self.loop.create_task(self.audio_track.add_new_bytes(chunk))
+                        audio_data += chunk
+
+
+            if audio_data:
+                await self._stream_audio_chunks(audio_data)
 
         except openai.APIError as e:
             self.emit("error", str(e))
         except Exception as e:
             self.emit("error", f"TTS synthesis failed: {str(e)}")
 
+    async def _stream_audio_chunks(self, audio_bytes: bytes) -> None:
+        """Stream audio data in chunks for smooth playback"""
+        chunk_size = int(OPENAI_TTS_SAMPLE_RATE * OPENAI_TTS_CHANNELS * 2 * 20 / 1000) 
+        
+        for i in range(0, len(audio_bytes), chunk_size):
+            chunk = audio_bytes[i:i + chunk_size]
+            
+            if len(chunk) < chunk_size and len(chunk) > 0:
+                padding_needed = chunk_size - len(chunk)
+                chunk += b'\x00' * padding_needed
+            
+            if len(chunk) == chunk_size:
+                self.loop.create_task(self.audio_track.add_new_bytes(chunk))
+                await asyncio.sleep(0.001)
+
     async def aclose(self) -> None:
         """Cleanup resources"""
         await self._client.close()

videosdk/plugins/openai/version.py

@@ -1 +1 @@
-__version__ = "0.0.19"
+__version__ = "0.0.21"

{videosdk_plugins_openai-0.0.19.dist-info → videosdk_plugins_openai-0.0.21.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: videosdk-plugins-openai
-Version: 0.0.19
+Version: 0.0.21
 Summary: VideoSDK Agent Framework plugin for OpenAI services
 Author: videosdk
 License-Expression: Apache-2.0
@@ -13,7 +13,7 @@ Classifier: Topic :: Multimedia :: Video
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Requires-Python: >=3.11
 Requires-Dist: openai[realtime]>=1.68.2
-Requires-Dist: videosdk-agents>=0.0.19
+Requires-Dist: videosdk-agents>=0.0.21
 Description-Content-Type: text/markdown
 
 # VideoSDK OpenAI Plugin

videosdk_plugins_openai-0.0.21.dist-info/RECORD

@@ -0,0 +1,9 @@
+videosdk/plugins/openai/__init__.py,sha256=1jbc4HOYxkLeruM9RAqmZYSBdnr74gnPHmCNMKXEPrg,259
+videosdk/plugins/openai/llm.py,sha256=h6xuJmyjg6InL9tr5pKBGt_5bNMpJ4XqnO72OtmCJ0c,7122
+videosdk/plugins/openai/realtime_api.py,sha256=B2RlEV_yK0R4K1dPTyhhPewoa9bzd43ytEfsLKaHUUQ,22554
+videosdk/plugins/openai/stt.py,sha256=YZROX-BjTqtWiT6ouMZacLkMYbmao3emB-88ewN93jg,9492
+videosdk/plugins/openai/tts.py,sha256=o5ktMUzjPkj64L5qqRaKPTWq7Na56TshMnLfU-sK36k,4417
+videosdk/plugins/openai/version.py,sha256=9_nCx08vWHyj8RiwIYqwqah3T4SPB46e0jnaNSxxfDc,22
+videosdk_plugins_openai-0.0.21.dist-info/METADATA,sha256=Docf55C8KnCxCAXOGa6i_UxbuYsLCPNiIyW25EdJEQM,827
+videosdk_plugins_openai-0.0.21.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+videosdk_plugins_openai-0.0.21.dist-info/RECORD,,

videosdk_plugins_openai-0.0.19.dist-info/RECORD

@@ -1,9 +0,0 @@
-videosdk/plugins/openai/__init__.py,sha256=1jbc4HOYxkLeruM9RAqmZYSBdnr74gnPHmCNMKXEPrg,259
-videosdk/plugins/openai/llm.py,sha256=wvWfQ8OA1AcdMMDEf4y93-2vSnSdulYjGpd9IVWFdxc,6145
-videosdk/plugins/openai/realtime_api.py,sha256=B2RlEV_yK0R4K1dPTyhhPewoa9bzd43ytEfsLKaHUUQ,22554
-videosdk/plugins/openai/stt.py,sha256=YZROX-BjTqtWiT6ouMZacLkMYbmao3emB-88ewN93jg,9492
-videosdk/plugins/openai/tts.py,sha256=LDsYXuHBoN-8g1iYt7JV_vRWOJZvhUN8QZQj_q264rU,3635
-videosdk/plugins/openai/version.py,sha256=9OK8yo7gxBYvFxF5sp_Pg19ucyUGzDpUXRaYqvJmYCQ,22
-videosdk_plugins_openai-0.0.19.dist-info/METADATA,sha256=ng3Wo9ut4Iua1rQBNHeDrA7dRupHxjThjvEKMzl7w-g,827
-videosdk_plugins_openai-0.0.19.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-videosdk_plugins_openai-0.0.19.dist-info/RECORD,,