videosdk-plugins-openai 0.0.22__py3-none-any.whl → 0.0.24__py3-none-any.whl

videosdk/plugins/openai/llm.py

@@ -133,7 +133,7 @@ class OpenAILLM(LLM):
                     tool_schema = build_openai_schema(tool)
                     formatted_tools.append(tool_schema)
                 except Exception as e:
-                    print(f"Failed to format tool {tool}: {e}")
+                    self.emit("error", f"Failed to format tool {tool}: {e}")
                     continue
             
             if formatted_tools:
@@ -167,7 +167,7 @@ class OpenAILLM(LLM):
                         args = json.loads(current_function_call["arguments"])
                         current_function_call["arguments"] = args
                     except json.JSONDecodeError:
-                        print(f"Failed to parse function arguments: {current_function_call['arguments']}")
+                        self.emit("error", f"Failed to parse function arguments: {current_function_call['arguments']}")
                         current_function_call["arguments"] = {}
                     
                     yield LLMResponse(

videosdk/plugins/openai/realtime_api.py

@@ -24,6 +24,8 @@ from videosdk.agents import (
     global_event_emitter,
     Agent
 )
+from videosdk.agents import realtime_metrics_collector
+
 
 load_dotenv()
 from openai.types.beta.realtime.session import InputAudioTranscription, TurnDetection
@@ -121,6 +123,7 @@ class OpenAIRealtime(RealtimeBaseModel[OpenAIEventTypes]):
         self.api_key = api_key or os.getenv("OPENAI_API_KEY")
         self.base_url = base_url or OPENAI_BASE_URL
         if not self.api_key:
+            self.emit("error", "OpenAI API key must be provided or set in OPENAI_API_KEY environment variable")
             raise ValueError("OpenAI API key must be provided or set in OPENAI_API_KEY environment variable")
         self._http_session: Optional[aiohttp.ClientSession] = None
         self._session: Optional[OpenAISession] = None
@@ -133,6 +136,7 @@ class OpenAIRealtime(RealtimeBaseModel[OpenAIEventTypes]):
         self.config: OpenAIRealtimeConfig = config or OpenAIRealtimeConfig()
         self.input_sample_rate = 48000
         self.target_sample_rate = 16000
+        self._agent_speaking = False
     
     def set_agent(self, agent: Agent) -> None:
         self._instructions = agent.instructions
@@ -202,6 +206,7 @@ class OpenAIRealtime(RealtimeBaseModel[OpenAIEventTypes]):
     async def create_response(self) -> None:
         """Create a response to the OpenAI realtime API"""
         if not self._session:
+            self.emit("error", "No active WebSocket session")
             raise RuntimeError("No active WebSocket session")
             
         response_event = {
@@ -245,15 +250,15 @@ class OpenAIRealtime(RealtimeBaseModel[OpenAIEventTypes]):
                 msg = await session.ws.receive()
                 
                 if msg.type == aiohttp.WSMsgType.CLOSED:
-                    print("WebSocket closed with reason:", msg.extra)
+                    self.emit("error", f"WebSocket closed with reason: {msg.extra}")
                     break
                 elif msg.type == aiohttp.WSMsgType.ERROR:
-                    print("WebSocket error:", msg.data)
+                    self.emit("error", f"WebSocket error: {msg.data}")
                     break
                 elif msg.type == aiohttp.WSMsgType.TEXT:
                     await self._handle_message(json.loads(msg.data))
         except Exception as e:
-            print("WebSocket receive error:", str(e))
+            self.emit("error", f"WebSocket receive error: {str(e)}")
         finally:
             await self._cleanup_session(session)
 
@@ -277,11 +282,14 @@ class OpenAIRealtime(RealtimeBaseModel[OpenAIEventTypes]):
             elif event_type == "response.content_part.added":
                 await self._handle_content_part_added(data)
                 
+            elif event_type == "response.text.delta":
+                await self._handle_text_delta(data)
+
             elif event_type == "response.audio.delta":
                 await self._handle_audio_delta(data)
                 
             elif event_type == "response.audio_transcript.delta":
-                await self._handle_transcript_delta(data)
+                await self._handle_audio_transcript_delta(data)
                 
             elif event_type == "response.done":
                 await self._handle_response_done(data)
@@ -314,10 +322,11 @@ class OpenAIRealtime(RealtimeBaseModel[OpenAIEventTypes]):
             await self.interrupt()
             if self.audio_track:
                 self.audio_track.interrupt()
+        await realtime_metrics_collector.set_user_speech_start()
 
     async def _handle_speech_stopped(self, data: dict) -> None:
         """Handle speech detection end"""
-        pass
+        await realtime_metrics_collector.set_user_speech_end()
 
     async def _handle_response_created(self, data: dict) -> None:
         """Handle initial response creation"""
@@ -339,6 +348,7 @@ class OpenAIRealtime(RealtimeBaseModel[OpenAIEventTypes]):
                         tool_info = get_tool_info(tool)
                         if tool_info.name == name:
                             try:
+                                await realtime_metrics_collector.add_tool_call(name)
                                 result = await tool(**arguments)
                                 await self.send_event({
                                     "type": "conversation.item.create",
@@ -361,26 +371,33 @@ class OpenAIRealtime(RealtimeBaseModel[OpenAIEventTypes]):
                                 })
                                 
                             except Exception as e:
-                                print(f"Error executing function {name}: {e}")
+                                self.emit("error", f"Error executing function {name}: {e}")
                             break
         except Exception as e:
-            print(f"Error handling output item done: {e}")
+            self.emit("error", f"Error handling output item done: {e}")
 
     async def _handle_content_part_added(self, data: dict) -> None:
         """Handle new content part"""
 
+    async def _handle_text_delta(self, data: dict) -> None:
+        """Handle text delta chunk"""
+        pass
+
     async def _handle_audio_delta(self, data: dict) -> None:
         """Handle audio chunk"""
         if "audio" not in self.config.modalities:
             return
             
         try:
+            if not self._agent_speaking:
+                await realtime_metrics_collector.set_agent_speech_start()
+                self._agent_speaking = True
             base64_audio_data = base64.b64decode(data.get("delta"))
             if base64_audio_data:
                 if self.audio_track and self.loop:
                     self.loop.create_task(self.audio_track.add_new_bytes(base64_audio_data))
         except Exception as e:
-            print(f"[ERROR] Error handling audio delta: {e}")
+            self.emit("error", f"Error handling audio delta: {e}")
             traceback.print_exc()
     
     async def interrupt(self) -> None:
@@ -391,18 +408,52 @@ class OpenAIRealtime(RealtimeBaseModel[OpenAIEventTypes]):
                 "event_id": str(uuid.uuid4())
             }
             await self.send_event(cancel_event)
+            await realtime_metrics_collector.set_interrupted()
         if self.audio_track:
             self.audio_track.interrupt()
+        if self._agent_speaking:
+            await realtime_metrics_collector.set_agent_speech_end(timeout=1.0)
+            self._agent_speaking = False
             
-    async def _handle_transcript_delta(self, data: dict) -> None:
+    async def _handle_audio_transcript_delta(self, data: dict) -> None:
         """Handle transcript chunk"""
-    
+        delta_content = data.get("delta", "")
+        if not hasattr(self, '_current_audio_transcript'):
+            self._current_audio_transcript = ""
+        self._current_audio_transcript += delta_content
+
     async def _handle_input_audio_transcription_completed(self, data: dict) -> None:
-        """Handle input audio transcription completion"""
+        """Handle input audio transcription completion for user transcript"""
+        transcript = data.get("transcript", "")
+        if transcript:
+            await realtime_metrics_collector.set_user_transcript(transcript)
+            try:
+                self.emit("realtime_model_transcription", {
+                    "role": "user",
+                    "text": transcript,
+                    "is_final": True
+                })
+            except Exception:
+                pass
 
     async def _handle_response_done(self, data: dict) -> None:
-        """Handle response completion"""
-    
+        """Handle response completion for agent transcript"""
+        if hasattr(self, '_current_audio_transcript') and self._current_audio_transcript:
+            await realtime_metrics_collector.set_agent_response(self._current_audio_transcript)
+            global_event_emitter.emit("text_response", {"text": self._current_audio_transcript, "type": "done"})
+            try:
+                self.emit("realtime_model_transcription", {
+                    "role": "agent",
+                    "text": self._current_audio_transcript,
+                    "is_final": True
+                })
+            except Exception:
+                pass
+            self._current_audio_transcript = ""
+        await realtime_metrics_collector.set_agent_speech_end(timeout=1.0)
+        self._agent_speaking = False
+        pass
+
     async def _handle_function_call_arguments_delta(self, data: dict) -> None:
         """Handle function call arguments delta"""
 
@@ -527,7 +578,7 @@ class OpenAIRealtime(RealtimeBaseModel[OpenAIEventTypes]):
                 tool_schema = build_openai_schema(tool)
                 oai_tools.append(tool_schema)
             except Exception as e:
-                print(f"Failed to format tool {tool}: {e}")
+                self.emit("error", f"Failed to format tool {tool}: {e}")
                 continue
                 
         return oai_tools
@@ -535,6 +586,7 @@ class OpenAIRealtime(RealtimeBaseModel[OpenAIEventTypes]):
     async def send_text_message(self, message: str) -> None:
         """Send a text message to the OpenAI realtime API"""
         if not self._session:
+            self.emit("error", "No active WebSocket session")
             raise RuntimeError("No active WebSocket session")
             
         await self.send_event({
@@ -552,11 +604,3 @@ class OpenAIRealtime(RealtimeBaseModel[OpenAIEventTypes]):
         })
         await self.create_response()
 
-    async def _handle_text_done(self, data: dict) -> None:
-        """Handle text response completion"""
-        try:
-            text_content = data.get("text", "")
-            if text_content:
-                global_event_emitter.emit("text_response", {"text": text_content, "type": "done"})
-        except Exception as e:
-            print(f"[ERROR] Error handling text done: {e}")

videosdk/plugins/openai/tts.py

@@ -36,6 +36,7 @@ class OpenAITTS(TTS):
         self.audio_track = None
         self.loop = None
         self.response_format = response_format
+        self._first_chunk_sent = False
         
         self.api_key = api_key or os.getenv("OPENAI_API_KEY")
         if not self.api_key:
@@ -55,6 +56,10 @@ class OpenAITTS(TTS):
                 ),
             ),
         )
+
+    def reset_first_audio_tracking(self) -> None:
+        """Reset the first audio tracking state for next TTS task"""
+        self._first_chunk_sent = False
     
     async def synthesize(
         self,
@@ -95,7 +100,6 @@ class OpenAITTS(TTS):
                     if chunk:
                         audio_data += chunk
 
-
             if audio_data:
                 await self._stream_audio_chunks(audio_data)
 
@@ -116,6 +120,10 @@ class OpenAITTS(TTS):
                 chunk += b'\x00' * padding_needed
             
             if len(chunk) == chunk_size:
+                if not self._first_chunk_sent and self._first_audio_callback:
+                    self._first_chunk_sent = True
+                    await self._first_audio_callback()
+                
                 self.loop.create_task(self.audio_track.add_new_bytes(chunk))
                 await asyncio.sleep(0.001)
 

videosdk/plugins/openai/version.py

@@ -1 +1 @@
-__version__ = "0.0.22"
+__version__ = "0.0.24"

{videosdk_plugins_openai-0.0.22.dist-info → videosdk_plugins_openai-0.0.24.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: videosdk-plugins-openai
-Version: 0.0.22
+Version: 0.0.24
 Summary: VideoSDK Agent Framework plugin for OpenAI services
 Author: videosdk
 License-Expression: Apache-2.0
@@ -13,7 +13,7 @@ Classifier: Topic :: Multimedia :: Video
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Requires-Python: >=3.11
 Requires-Dist: openai[realtime]>=1.68.2
-Requires-Dist: videosdk-agents>=0.0.22
+Requires-Dist: videosdk-agents>=0.0.24
 Description-Content-Type: text/markdown
 
 # VideoSDK OpenAI Plugin

videosdk_plugins_openai-0.0.24.dist-info/RECORD

@@ -0,0 +1,9 @@
+videosdk/plugins/openai/__init__.py,sha256=1jbc4HOYxkLeruM9RAqmZYSBdnr74gnPHmCNMKXEPrg,259
+videosdk/plugins/openai/llm.py,sha256=igKq1LRrJfgrIbhVFik8aJp1Cux5069sAX-tusfCg6k,7148
+videosdk/plugins/openai/realtime_api.py,sha256=s73iBlZE5bo1vDdnYOYw9VVE_0aliFJwUv4yEjxDBhE,24854
+videosdk/plugins/openai/stt.py,sha256=YZROX-BjTqtWiT6ouMZacLkMYbmao3emB-88ewN93jg,9492
+videosdk/plugins/openai/tts.py,sha256=m-15GslICL9dOa_H7YqIHP5ifif2OL-7DeTRQunQs9A,4814
+videosdk/plugins/openai/version.py,sha256=sE45w-zCTIpVQyXYRbP5E390wIU6bNzRFuAh6ySoc1w,22
+videosdk_plugins_openai-0.0.24.dist-info/METADATA,sha256=P7ontFkdhIuKtS_JjxdTTGtQWdtFoxphiRC66k4CtKA,827
+videosdk_plugins_openai-0.0.24.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+videosdk_plugins_openai-0.0.24.dist-info/RECORD,,

videosdk_plugins_openai-0.0.22.dist-info/RECORD

@@ -1,9 +0,0 @@
-videosdk/plugins/openai/__init__.py,sha256=1jbc4HOYxkLeruM9RAqmZYSBdnr74gnPHmCNMKXEPrg,259
-videosdk/plugins/openai/llm.py,sha256=h6xuJmyjg6InL9tr5pKBGt_5bNMpJ4XqnO72OtmCJ0c,7122
-videosdk/plugins/openai/realtime_api.py,sha256=WSzDWHcCQC8QsKLDmA5mm_oSN8UIHYMplesNliV5eUc,22611
-videosdk/plugins/openai/stt.py,sha256=YZROX-BjTqtWiT6ouMZacLkMYbmao3emB-88ewN93jg,9492
-videosdk/plugins/openai/tts.py,sha256=o5ktMUzjPkj64L5qqRaKPTWq7Na56TshMnLfU-sK36k,4417
-videosdk/plugins/openai/version.py,sha256=NoiGDztYD4fsDDnfSPiSzRkknkNHhFUtKZj0mhQiTYM,22
-videosdk_plugins_openai-0.0.22.dist-info/METADATA,sha256=9BJRuTdobykpCbIf5Gwr33z074lZjp-tCjdgBn5GUqg,827
-videosdk_plugins_openai-0.0.22.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-videosdk_plugins_openai-0.0.22.dist-info/RECORD,,