videosdk-plugins-openai 0.0.5__py3-none-any.whl → 0.0.6__py3-none-any.whl

videosdk/plugins/openai/__init__.py

@@ -1,6 +1,12 @@
 from .realtime_api import OpenAIRealtime, OpenAIRealtimeConfig
+from .llm import OpenAILLM
+from .stt import OpenAISTT
+from .tts import OpenAITTS
 
 __all__ = [
     'OpenAIRealtime',
-    'OpenAIRealtimeConfig'
+    'OpenAIRealtimeConfig',
+    'OpenAILLM',
+    'OpenAISTT',
+    'OpenAITTS',
 ]

videosdk/plugins/openai/llm.py

@@ -0,0 +1,161 @@
+from __future__ import annotations
+
+import os
+from typing import Any, AsyncIterator
+import json
+
+import httpx
+import openai
+from videosdk.agents import LLM, LLMResponse, ChatContext, ChatRole, ChatMessage, FunctionCall, FunctionCallOutput, ToolChoice, FunctionTool, is_function_tool, build_openai_schema
+
+class OpenAILLM(LLM):
+    
+    def __init__(
+        self,
+        *,
+        model: str = "gpt-4o",
+        api_key: str | None = None,
+        base_url: str | None = None,
+        temperature: float = 0.7,
+        tool_choice: ToolChoice = "auto",
+        max_completion_tokens: int | None = None,
+    ) -> None:
+        super().__init__()
+        self.api_key = api_key or os.getenv("OPENAI_API_KEY")
+        if not self.api_key:
+            raise ValueError("OpenAI API key must be provided either through api_key parameter or OPENAI_API_KEY environment variable")
+        
+        self.model = model
+        self.temperature = temperature
+        self.tool_choice = tool_choice
+        self.max_completion_tokens = max_completion_tokens
+        
+        self._client = openai.AsyncOpenAI(
+            api_key=self.api_key,
+            base_url=base_url or None,
+            max_retries=0,
+            http_client=httpx.AsyncClient(
+                timeout=httpx.Timeout(connect=15.0, read=5.0, write=5.0, pool=5.0),
+                follow_redirects=True,
+                limits=httpx.Limits(
+                    max_connections=50,
+                    max_keepalive_connections=50,
+                    keepalive_expiry=120,
+                ),
+            ),
+        )
+
+    async def chat(
+        self,
+        messages: ChatContext,
+        tools: list[FunctionTool] | None = None,
+        **kwargs: Any
+    ) -> AsyncIterator[LLMResponse]:
+        """
+        Implement chat functionality using OpenAI's chat completion API
+        
+        Args:
+            messages: ChatContext containing conversation history
+            tools: Optional list of function tools available to the model
+            **kwargs: Additional arguments passed to the OpenAI API
+            
+        Yields:
+            LLMResponse objects containing the model's responses
+        """
+        completion_params = {
+            "model": self.model,
+            "messages": [
+                {
+                    "role": msg.role.value,
+                    "content": msg.content,
+                    **({"name": msg.name} if hasattr(msg, 'name') else {})
+                } if isinstance(msg, ChatMessage) else
+                {
+                    "role": "assistant",
+                    "content": None,
+                    "function_call": {
+                        "name": msg.name,
+                        "arguments": msg.arguments
+                    }
+                } if isinstance(msg, FunctionCall) else
+                {
+                    "role": "function",
+                    "name": msg.name,
+                    "content": msg.output
+                } if isinstance(msg, FunctionCallOutput) else None
+                for msg in messages.items
+                if msg is not None 
+            ],
+            "temperature": self.temperature,
+            "stream": True,
+            "max_tokens": self.max_completion_tokens,
+        }
+
+        if tools:
+            formatted_tools = []
+            for tool in tools:
+                if not is_function_tool(tool):
+                    continue
+                try:
+                    tool_schema = build_openai_schema(tool)
+                    formatted_tools.append(tool_schema)
+                except Exception as e:
+                    print(f"Failed to format tool {tool}: {e}")
+                    continue
+            
+            if formatted_tools:
+                completion_params["functions"] = formatted_tools
+                completion_params["function_call"] = self.tool_choice
+
+        completion_params.update(kwargs)
+        try:
+            response_stream = await self._client.chat.completions.create(**completion_params)
+            current_content = ""
+            current_function_call = None
+
+            async for chunk in response_stream:
+                if not chunk.choices:
+                    continue
+                    
+                delta = chunk.choices[0].delta
+                if delta.function_call:
+                    if current_function_call is None:
+                        current_function_call = {
+                            "name": delta.function_call.name or "",
+                            "arguments": delta.function_call.arguments or ""
+                        }
+                    else:
+                        if delta.function_call.name:
+                            current_function_call["name"] += delta.function_call.name
+                        if delta.function_call.arguments:
+                            current_function_call["arguments"] += delta.function_call.arguments
+                elif current_function_call is not None:  
+                    try:
+                        args = json.loads(current_function_call["arguments"])
+                        current_function_call["arguments"] = args
+                    except json.JSONDecodeError:
+                        print(f"Failed to parse function arguments: {current_function_call['arguments']}")
+                        current_function_call["arguments"] = {}
+                    
+                    yield LLMResponse(
+                        content="",
+                        role=ChatRole.ASSISTANT,
+                        metadata={"function_call": current_function_call}
+                    )
+                    current_function_call = None
+                
+                elif delta.content is not None:
+                    current_content += delta.content
+                    yield LLMResponse(
+                        content=current_content,
+                        role=ChatRole.ASSISTANT
+                    )
+
+        except Exception as e:
+            self.emit("error", e)
+            raise
+
+    async def aclose(self) -> None:
+        """Cleanup resources by closing the HTTP client"""
+        if self._client:
+            await self._client.close() 

videosdk/plugins/openai/realtime_api.py

@@ -10,6 +10,8 @@ from dotenv import load_dotenv
 import uuid
 import base64
 import aiohttp
+import numpy as np
+from scipy import signal
 import traceback
 from videosdk.agents import (
     FunctionTool,
@@ -26,8 +28,6 @@ load_dotenv()
 from openai.types.beta.realtime.session import InputAudioTranscription, TurnDetection
 
 OPENAI_BASE_URL = "https://api.openai.com/v1"
-SAMPLE_RATE = 24000
-NUM_CHANNELS = 1
 
 DEFAULT_TEMPERATURE = 0.8
 DEFAULT_TURN_DETECTION = TurnDetection(
@@ -130,9 +130,12 @@ class OpenAIRealtime(RealtimeBaseModel[OpenAIEventTypes]):
         self.audio_track: Optional[CustomAudioStreamTrack] = None
         self._formatted_tools: Optional[List[Dict[str, Any]]] = None
         self.config: OpenAIRealtimeConfig = config or OpenAIRealtimeConfig()
-        # self.on("instructions_updated", self._handle_instructions_updated)
-        # self.on("tools_updated", self._handle_tools_updated) 
-
+        # global_event_emitter.on("instructions_updated", self._handle_instructions_updated)
+        # global_event_emitter.on("tools_updated", self._handle_tools_updated)
+        
+        self.input_sample_rate = 48000
+        self.target_sample_rate = 16000
+    
     def set_agent(self, agent: Agent) -> None:
         self._instructions = agent.instructions
         self._tools = agent.tools
@@ -153,6 +156,9 @@ class OpenAIRealtime(RealtimeBaseModel[OpenAIEventTypes]):
     async def handle_audio_input(self, audio_data: bytes) -> None:
         """Handle incoming audio data from the user"""
         if self._session and not self._closing and "audio" in self.config.modalities:
+            audio_data = np.frombuffer(audio_data, dtype=np.int16)
+            audio_data = signal.resample(audio_data, int(len(audio_data) * self.target_sample_rate / self.input_sample_rate))
+            audio_data = audio_data.astype(np.int16).tobytes()
             base64_audio_data = base64.b64encode(audio_data).decode("utf-8")
             audio_event = {
                 "type": "input_audio_buffer.append",

videosdk/plugins/openai/stt.py

@@ -0,0 +1,260 @@
+from __future__ import annotations
+
+import asyncio
+import base64
+import os
+from typing import Any, Optional
+from urllib.parse import urlencode
+from scipy import signal
+import aiohttp
+import httpx
+import openai
+import numpy as np
+from videosdk.agents import STT as BaseSTT, STTResponse, SpeechEventType, SpeechData, global_event_emitter
+
+class OpenAISTT(BaseSTT):
+    def __init__(
+        self,
+        *,
+        api_key: str,
+        model: str = "whisper-1",
+        base_url: str | None = None,
+        prompt: str | None = None,
+        language: str = "en",
+        turn_detection: dict | None = None,
+    ) -> None:
+        super().__init__()
+        
+        self.api_key = api_key or os.getenv("OPENAI_API_KEY")
+        if not self.api_key:
+            raise ValueError("OpenAI API key must be provided either through api_key parameter or OPENAI_API_KEY environment variable")
+        
+        self.model = model
+        self.language = language
+        self.prompt = prompt
+        self.turn_detection = turn_detection or {
+            "type": "server_vad",
+            "threshold": 0.5,
+            "prefix_padding_ms": 300,
+            "silence_duration_ms": 500,
+        }
+        
+        self.client = openai.AsyncClient(
+            max_retries=0,
+            api_key=api_key,
+            base_url=base_url or None,
+            http_client=httpx.AsyncClient(
+                timeout=httpx.Timeout(connect=15.0, read=5.0, write=5.0, pool=5.0),
+                follow_redirects=True,
+                limits=httpx.Limits(
+                    max_connections=50,
+                    max_keepalive_connections=50,
+                    keepalive_expiry=120,
+                ),
+            ),
+        )
+        
+        self._session: Optional[aiohttp.ClientSession] = None
+        self._ws: Optional[aiohttp.ClientWebSocketResponse] = None
+        self._ws_task: Optional[asyncio.Task] = None
+        self._current_text = ""
+        self._last_interim_at = 0
+    
+        self.input_sample_rate = 48000
+        self.target_sample_rate = 16000
+        
+    async def process_audio(
+        self,
+        audio_frames: bytes,
+        language: Optional[str] = None,
+        **kwargs: Any
+    ) -> None:
+        """Process audio frames and send to OpenAI's Realtime API"""
+        
+        if not self._ws:
+            await self._connect_ws()
+            self._ws_task = asyncio.create_task(self._listen_for_responses())
+            
+        try:
+            audio_data = np.frombuffer(audio_frames, dtype=np.int16)
+            audio_data = signal.resample(audio_data, int(len(audio_data) * self.target_sample_rate / self.input_sample_rate))
+            audio_data = audio_data.astype(np.int16).tobytes()
+            audio_data = base64.b64encode(audio_data).decode("utf-8")
+            message = {
+                "type": "input_audio_buffer.append",
+                "audio": audio_data,
+            }
+            await self._ws.send_json(message)
+        except Exception as e:
+            print(f"Error in process_audio: {str(e)}")
+            self.emit("error", str(e))
+            if self._ws:
+                await self._ws.close()
+                self._ws = None
+                if self._ws_task:
+                    self._ws_task.cancel()
+                    self._ws_task = None
+
+    async def _listen_for_responses(self) -> None:
+        """Background task to listen for WebSocket responses"""
+        if not self._ws:
+            return
+            
+        try:
+            async for msg in self._ws:
+                if msg.type == aiohttp.WSMsgType.TEXT:
+                    data = msg.json()
+                    responses = self._handle_ws_message(data)
+                    for response in responses:
+                        if self._transcript_callback:
+                            await self._transcript_callback(response)
+                elif msg.type == aiohttp.WSMsgType.ERROR:
+                    error = f"WebSocket error: {self._ws.exception()}"
+                    print(error)
+                    self.emit("error", error)
+                    break
+                elif msg.type == aiohttp.WSMsgType.CLOSED:
+                    print("WebSocket connection closed")
+                    break
+        except Exception as e:
+            error = f"Error in WebSocket listener: {str(e)}"
+            print(error)
+            self.emit("error", error)
+        finally:
+            if self._ws:
+                await self._ws.close()
+                self._ws = None
+                
+    async def _connect_ws(self) -> None:
+        """Establish WebSocket connection with OpenAI's Realtime API"""
+        
+        if not self._session:
+            self._session = aiohttp.ClientSession()
+            
+        config = {
+            "type": "transcription_session.update",
+            "session": {
+                "input_audio_format": "pcm16",
+                "input_audio_transcription": {
+                    "model": self.model,
+                    "prompt": self.prompt or "",
+                    "language": self.language if self.language else None,
+                },
+                "turn_detection": self.turn_detection,
+                "input_audio_noise_reduction": {
+                    "type": "near_field"
+                },
+                "include": ["item.input_audio_transcription.logprobs"]
+            }
+        }
+        
+        query_params = {
+            "intent": "transcription",
+        }
+        headers = {
+            "User-Agent": "VideoSDK",
+            "Authorization": f"Bearer {self.api_key}",
+            "OpenAI-Beta": "realtime=v1",
+        }
+        
+        base_url = str(self.client.base_url).rstrip('/')
+        ws_url = f"{base_url}/realtime?{urlencode(query_params)}"
+        if ws_url.startswith("http"):
+            ws_url = ws_url.replace("http", "ws", 1)
+
+        try:
+            self._ws = await self._session.ws_connect(ws_url, headers=headers)
+            
+            initial_response = await self._ws.receive_json()
+            
+            if initial_response.get("type") != "transcription_session.created":
+                raise Exception(f"Expected session creation, got: {initial_response}")
+            
+            await self._ws.send_json(config)
+            
+            update_response = await self._ws.receive_json()
+            
+            if update_response.get("type") != "transcription_session.updated":
+                raise Exception(f"Configuration update failed: {update_response}")
+            
+        except Exception as e:
+            print(f"Error connecting to WebSocket: {str(e)}")
+            if self._ws:
+                await self._ws.close()
+                self._ws = None
+            raise
+        
+    def _handle_ws_message(self, msg: dict) -> list[STTResponse]:
+        """Handle incoming WebSocket messages and generate STT responses"""
+        responses = []
+        
+        try:
+            msg_type = msg.get("type")
+            if msg_type == "conversation.item.input_audio_transcription.delta":
+                delta = msg.get("delta", "")
+                if delta:
+                    self._current_text += delta
+                    current_time = asyncio.get_event_loop().time()
+                    
+                    if current_time - self._last_interim_at > 0.5:
+                        responses.append(STTResponse(
+                            event_type=SpeechEventType.INTERIM,
+                            data=SpeechData(
+                                text=self._current_text,
+                                language=self.language,
+                            ),
+                            metadata={"model": self.model}
+                        ))
+                        self._last_interim_at = current_time
+                        
+            elif msg_type == "conversation.item.input_audio_transcription.completed":
+                transcript = msg.get("transcript", "")
+                if transcript:
+                    responses.append(STTResponse(
+                        event_type=SpeechEventType.FINAL,
+                        data=SpeechData(
+                            text=transcript,
+                            language=self.language,
+                        ),
+                        metadata={"model": self.model}
+                    ))
+                    self._current_text = ""
+            
+            elif msg_type == "input_audio_buffer.speech_started":
+                global_event_emitter.emit("speech_started")
+            
+            elif msg_type == "input_audio_buffer.speech_stopped":
+                global_event_emitter.emit("speech_stopped")
+                
+        except Exception as e:
+            print(f"Error handling WebSocket message: {str(e)}")
+        
+        return responses
+
+    async def aclose(self) -> None:
+        """Cleanup resources"""
+        if self._ws_task:
+            self._ws_task.cancel()
+            try:
+                await self._ws_task
+            except asyncio.CancelledError:
+                pass
+            self._ws_task = None
+            
+        if self._ws:
+            await self._ws.close()
+            self._ws = None
+            
+        if self._session:
+            await self._session.close()
+            self._session = None
+            
+        await self.client.close()
+
+    async def _ensure_ws_connection(self):
+        """Ensure WebSocket is connected, reconnect if necessary"""
+        if not self._ws or self._ws.closed:
+            await self._connect_ws()
+
+
+

videosdk/plugins/openai/tts.py

@@ -0,0 +1,109 @@
+from __future__ import annotations
+
+from typing import Any, AsyncIterator, Literal, Optional, Union
+import httpx
+import os
+import openai
+
+from videosdk.agents import TTS
+
+OPENAI_TTS_SAMPLE_RATE = 24000
+OPENAI_TTS_CHANNELS = 1
+
+DEFAULT_MODEL = "gpt-4o-mini-tts"
+DEFAULT_VOICE = "ash"
+_RESPONSE_FORMATS = Union[Literal["mp3", "opus", "aac", "flac", "wav", "pcm"], str]
+
+class OpenAITTS(TTS):
+    def __init__(
+        self,
+        *,
+        model: str = DEFAULT_MODEL,
+        voice: str = DEFAULT_VOICE,
+        speed: float = 1.0,
+        instructions: str | None = None,
+        api_key: str | None = None,
+        base_url: str | None = None,
+        response_format: str = "pcm"
+    ) -> None:
+        super().__init__(sample_rate=OPENAI_TTS_SAMPLE_RATE, num_channels=OPENAI_TTS_CHANNELS)
+        
+        self.model = model
+        self.voice = voice
+        self.speed = speed
+        self.instructions = instructions
+        self.audio_track = None
+        self.loop = None
+        self.response_format = response_format
+        
+        self.api_key = api_key or os.getenv("OPENAI_API_KEY")
+        if not self.api_key:
+            raise ValueError("OpenAI API key must be provided either through api_key parameter or OPENAI_API_KEY environment variable")
+        
+        self._client = openai.AsyncClient(
+            max_retries=0,
+            api_key=self.api_key,
+            base_url=base_url or None,
+            http_client=httpx.AsyncClient(
+                timeout=httpx.Timeout(connect=15.0, read=5.0, write=5.0, pool=5.0),
+                follow_redirects=True,
+                limits=httpx.Limits(
+                    max_connections=50,
+                    max_keepalive_connections=50,
+                    keepalive_expiry=120,
+                ),
+            ),
+        )
+    
+    async def synthesize(
+        self,
+        text: AsyncIterator[str] | str,
+        voice_id: Optional[str] = None,
+        **kwargs: Any
+    ) -> None:
+        """
+        Convert text to speech using OpenAI's TTS API and stream to audio track
+        
+        Args:
+            text: Text to convert to speech
+            voice_id: Optional voice override
+            **kwargs: Additional provider-specific arguments
+        """
+        try:
+            if isinstance(text, AsyncIterator):
+                full_text = ""
+                async for chunk in text:
+                    full_text += chunk
+            else:
+                full_text = text
+
+            if not self.audio_track or not self.loop:
+                self.emit("error", "Audio track or event loop not set")
+                return
+
+            async with self._client.audio.speech.with_streaming_response.create(
+                model=self.model,
+                voice=voice_id or self.voice,
+                input=full_text,
+                speed=self.speed,
+                response_format=self.response_format,
+                **({"instructions": self.instructions} if self.instructions else {})
+            ) as response:
+                async for chunk in response.iter_bytes():
+                    if chunk:
+                        self.loop.create_task(self.audio_track.add_new_bytes(chunk))
+
+        except openai.APIError as e:
+            self.emit("error", str(e))
+        except Exception as e:
+            self.emit("error", f"TTS synthesis failed: {str(e)}")
+
+    async def aclose(self) -> None:
+        """Cleanup resources"""
+        await self._client.close()
+        await super().aclose()
+
+    async def interrupt(self) -> None:
+        """Interrupt the TTS process"""
+        if self.audio_track:
+            self.audio_track.interrupt()

videosdk/plugins/openai/version.py

@@ -1 +1 @@
-__version__ = "0.0.5"
+__version__ = "0.0.6"

{videosdk_plugins_openai-0.0.5.dist-info → videosdk_plugins_openai-0.0.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: videosdk-plugins-openai
-Version: 0.0.5
+Version: 0.0.6
 Summary: VideoSDK Agent Framework plugin for OpenAI services
 Author: videosdk
 Keywords: ai,audio,openai,video,videosdk
@@ -12,7 +12,7 @@ Classifier: Topic :: Multimedia :: Video
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Requires-Python: >=3.11
 Requires-Dist: openai[realtime]>=1.68.2
-Requires-Dist: videosdk-agents>=0.0.8
+Requires-Dist: videosdk-agents>=0.0.9
 Description-Content-Type: text/markdown
 
 VideoSDK OpenAI Plugin

videosdk_plugins_openai-0.0.6.dist-info/RECORD

@@ -0,0 +1,9 @@
+videosdk/plugins/openai/__init__.py,sha256=1jbc4HOYxkLeruM9RAqmZYSBdnr74gnPHmCNMKXEPrg,259
+videosdk/plugins/openai/llm.py,sha256=wvWfQ8OA1AcdMMDEf4y93-2vSnSdulYjGpd9IVWFdxc,6145
+videosdk/plugins/openai/realtime_api.py,sha256=3qPTTGrw8P3TnHva2898mAlSjUEbbifSxKZq3blltd4,23802
+videosdk/plugins/openai/stt.py,sha256=YZROX-BjTqtWiT6ouMZacLkMYbmao3emB-88ewN93jg,9492
+videosdk/plugins/openai/tts.py,sha256=LDsYXuHBoN-8g1iYt7JV_vRWOJZvhUN8QZQj_q264rU,3635
+videosdk/plugins/openai/version.py,sha256=CzcXzRPJNqTNVzWp48bKd5k9CaHSjpVkLYKkiQcGpAY,21
+videosdk_plugins_openai-0.0.6.dist-info/METADATA,sha256=pAfpKWgYGGIDevroF_Y0V3_ztEY0BPc0If0uy7E1syQ,774
+videosdk_plugins_openai-0.0.6.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+videosdk_plugins_openai-0.0.6.dist-info/RECORD,,

videosdk_plugins_openai-0.0.5.dist-info/RECORD

@@ -1,6 +0,0 @@
-videosdk/plugins/openai/__init__.py,sha256=lytPalXAu1WWdr08gzLWiz6yGkBvupeQJlcvR0sGAj8,126
-videosdk/plugins/openai/realtime_api.py,sha256=lOvZ29nNxJymfUWJR889Cil7TYN5n5Ww7ACZVJQGXpY,23416
-videosdk/plugins/openai/version.py,sha256=v0zqRembIW__d7HVDd71J2eoXGtALnzdY8JdBVk-BPY,21
-videosdk_plugins_openai-0.0.5.dist-info/METADATA,sha256=8sugMkdnWMt4AfJfZfQAcpkC8tPJwoVKBBPSo0IEeAU,774
-videosdk_plugins_openai-0.0.5.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-videosdk_plugins_openai-0.0.5.dist-info/RECORD,,