isa-model 0.3.91__py3-none-any.whl → 0.4.0__py3-none-any.whl

isa_model/inference/services/audio/openai_realtime_service.py

@@ -1,57 +1,67 @@
 import logging
 import json
 import asyncio
+import base64
 from typing import Dict, Any, List, Optional, Callable, AsyncGenerator
 import aiohttp
 from tenacity import retry, stop_after_attempt, wait_exponential
 
-from isa_model.inference.services.base_service import BaseService
-from isa_model.inference.providers.base_provider import BaseProvider
-from isa_model.inference.billing_tracker import ServiceType
+from isa_model.inference.services.audio.base_realtime_service import BaseRealtimeService, RealtimeEventType
+from isa_model.core.types import ServiceType
 
 logger = logging.getLogger(__name__)
 
-class OpenAIRealtimeService(BaseService):
+class OpenAIRealtimeService(BaseRealtimeService):
     """
     OpenAI Realtime API service for real-time audio conversations.
     Uses gpt-4o-mini-realtime-preview model for interactive audio chat.
     """
     
-    def __init__(self, provider: 'BaseProvider', model_name: str = "gpt-4o-mini-realtime-preview"):
-        super().__init__(provider, model_name)
+    def __init__(self, provider_name: str = "openai", model_name: str = "gpt-4o-realtime-preview-2024-10-01", **kwargs):
+        super().__init__(provider_name, model_name, **kwargs)
         
-        self.api_key = self.config.get('api_key')
-        self.base_url = self.config.get('api_base', 'https://api.openai.com/v1')
+        provider_config = self.get_provider_config()
+        self.api_key = provider_config.get('api_key') or self.get_api_key()
+        self.base_url = provider_config.get('api_base_url', 'https://api.openai.com/v1')
+        self.websocket_url = f"wss://api.openai.com/v1/realtime?model={self.model_name}"
         
-        # Default session configuration
+        # Default session configuration based on latest API
         self.default_config = {
-            "model": self.model_name,
-            "modalities": ["audio", "text"],
+            "modalities": ["text", "audio"],
+            "instructions": "You are a helpful assistant.",
             "voice": "alloy",
             "input_audio_format": "pcm16",
             "output_audio_format": "pcm16",
             "input_audio_transcription": {
                 "model": "whisper-1"
             },
-            "turn_detection": None,
+            "turn_detection": {
+                "type": "server_vad",
+                "threshold": 0.5,
+                "prefix_padding_ms": 300,
+                "silence_duration_ms": 200
+            },
             "tools": [],
-            "tool_choice": "none",
-            "temperature": 0.7,
-            "max_response_output_tokens": 200,
-            "speed": 1.1,
-            "tracing": "auto"
+            "tool_choice": "auto",
+            "temperature": 0.8,
+            "max_response_output_tokens": "inf"
+        }
+        
+        # Session limits based on API documentation
+        self.session_limits = {
+            "max_context_tokens": 128000,
+            "max_session_time_minutes": 15,
+            "audio_tokens_per_minute": 800
         }
         
         logger.info(f"Initialized OpenAIRealtimeService with model '{self.model_name}'")
+        
+        # Add default event handlers for common events
+        self._setup_default_handlers()
     
-    @retry(
-        stop=stop_after_attempt(3),
-        wait=wait_exponential(multiplier=1, min=4, max=10),
-        reraise=True
-    )
     async def create_session(
         self, 
-        instructions: str = "You are a friendly assistant.",
+        instructions: str = "You are a helpful assistant.",
         modalities: Optional[List[str]] = None,
         voice: str = "alloy",
         **kwargs
@@ -62,80 +72,102 @@ class OpenAIRealtimeService(BaseService):
             session_config = self.default_config.copy()
             session_config.update({
                 "instructions": instructions,
-                "modalities": modalities if modalities is not None else ["audio", "text"],
+                "modalities": modalities if modalities is not None else ["text", "audio"],
                 "voice": voice,
                 **kwargs
             })
             
-            # Create session via REST API
-            url = f"{self.base_url}/realtime/sessions"
-            headers = {
-                "Authorization": f"Bearer {self.api_key}",
-                "Content-Type": "application/json"
-            }
+            # Store session config for WebSocket connection
+            self.session_config = session_config
             
-            async with aiohttp.ClientSession() as session:
-                async with session.post(url, headers=headers, json=session_config) as response:
-                    if response.status == 200:
-                        result = await response.json()
-                        
-                        # Track usage for billing
-                        self._track_usage(
-                            service_type=ServiceType.AUDIO_STT,  # Realtime combines STT/TTS
-                            operation="create_session",
-                            metadata={
-                                "session_id": result.get("id"),
-                                "model": self.model_name,
-                                "modalities": session_config["modalities"]
-                            }
-                        )
-                        
-                        return result
-                    else:
-                        error_text = await response.text()
-                        raise Exception(f"Failed to create session: {response.status} - {error_text}")
+            # Generate a session ID (WebSocket-based, no REST endpoint)
+            import uuid
+            self.session_id = str(uuid.uuid4())
+            
+            # Track session creation for billing
+            await self._track_usage(
+                service_type=ServiceType.AUDIO_REALTIME,
+                operation="create_session",
+                metadata={
+                    "session_id": self.session_id,
+                    "model": self.model_name,
+                    "modalities": session_config["modalities"]
+                }
+            )
+            
+            return {
+                "id": self.session_id,
+                "model": self.model_name,
+                "modalities": session_config["modalities"],
+                "instructions": instructions,
+                "voice": voice,
+                "status": "created"
+            }
                         
         except Exception as e:
             logger.error(f"Error creating realtime session: {e}")
             raise
     
-    async def connect_websocket(self, session_id: str) -> aiohttp.ClientWebSocketResponse:
-        """Connect to the realtime WebSocket for a session"""
+    async def connect_websocket(self, **kwargs) -> bool:
+        """Connect to the realtime WebSocket"""
         try:
-            ws_url = f"wss://api.openai.com/v1/realtime/sessions/{session_id}/ws"
             headers = {
                 "Authorization": f"Bearer {self.api_key}",
                 "OpenAI-Beta": "realtime=v1"
             }
             
-            session = aiohttp.ClientSession()
-            ws = await session.ws_connect(ws_url, headers=headers)
+            self.client_session = aiohttp.ClientSession()
+            self.websocket = await self.client_session.ws_connect(
+                self.websocket_url, 
+                headers=headers
+            )
+            
+            # Send session.update event to configure the session
+            if hasattr(self, 'session_config'):
+                await self._send_event({
+                    "type": "session.update",
+                    "session": self.session_config
+                })
             
-            logger.info(f"Connected to realtime WebSocket for session {session_id}")
-            return ws
+            self.is_connected = True
+            logger.info(f"Connected to realtime WebSocket with model {self.model_name}")
+            return True
             
         except Exception as e:
             logger.error(f"Error connecting to WebSocket: {e}")
+            self.is_connected = False
             raise
     
     async def send_audio_message(
         self, 
-        ws: aiohttp.ClientWebSocketResponse, 
         audio_data: bytes,
-        format: str = "pcm16"
-    ):
+        format: str = "pcm16",
+        **kwargs
+    ) -> Dict[str, Any]:
         """Send audio data to the realtime session"""
         try:
-            message = {
-                "type": "input_audio_buffer.append",
-                "audio": audio_data.hex() if format == "pcm16" else audio_data
-            }
+            if not self.is_connected or not self.websocket:
+                raise RuntimeError("WebSocket not connected")
+                
+            # Convert audio data to base64
+            audio_base64 = base64.b64encode(audio_data).decode('utf-8')
             
-            await ws.send_str(json.dumps(message))
+            # Send audio buffer append event
+            await self._send_event({
+                "type": RealtimeEventType.INPUT_AUDIO_BUFFER_APPEND.value,
+                "audio": audio_base64
+            })
             
             # Commit the audio buffer
-            commit_message = {"type": "input_audio_buffer.commit"}
-            await ws.send_str(json.dumps(commit_message))
+            await self._send_event({
+                "type": RealtimeEventType.INPUT_AUDIO_BUFFER_COMMIT.value
+            })
+            
+            return {
+                "status": "sent",
+                "audio_size_bytes": len(audio_data),
+                "format": format
+            }
             
         except Exception as e:
             logger.error(f"Error sending audio message: {e}")
@@ -143,13 +175,17 @@ class OpenAIRealtimeService(BaseService):
     
     async def send_text_message(
         self, 
-        ws: aiohttp.ClientWebSocketResponse, 
-        text: str
-    ):
+        text: str,
+        **kwargs
+    ) -> Dict[str, Any]:
         """Send text message to the realtime session"""
         try:
-            message = {
-                "type": "conversation.item.create",
+            if not self.is_connected or not self.websocket:
+                raise RuntimeError("WebSocket not connected")
+                
+            # Create conversation item
+            await self._send_event({
+                "type": RealtimeEventType.CONVERSATION_ITEM_CREATE.value,
                 "item": {
                     "type": "message",
                     "role": "user",
@@ -160,75 +196,115 @@ class OpenAIRealtimeService(BaseService):
                         }
                     ]
                 }
-            }
+            })
             
-            await ws.send_str(json.dumps(message))
+            # Trigger response creation
+            await self._send_event({
+                "type": RealtimeEventType.RESPONSE_CREATE.value
+            })
             
-            # Trigger response
-            response_message = {"type": "response.create"}
-            await ws.send_str(json.dumps(response_message))
+            return {
+                "status": "sent",
+                "text": text,
+                "message_length": len(text)
+            }
             
         except Exception as e:
             logger.error(f"Error sending text message: {e}")
             raise
     
     async def listen_for_responses(
-        self, 
-        ws: aiohttp.ClientWebSocketResponse,
-        message_handler: Optional[Callable] = None
+        self,
+        message_handler: Optional[Callable] = None,
+        **kwargs
     ) -> AsyncGenerator[Dict[str, Any], None]:
         """Listen for responses from the realtime session"""
         try:
-            async for msg in ws:
+            if not self.is_connected or not self.websocket:
+                raise RuntimeError("WebSocket not connected")
+                
+            async for msg in self.websocket:
                 if msg.type == aiohttp.WSMsgType.TEXT:
                     try:
-                        data = json.loads(msg.data)
+                        event = json.loads(msg.data)
+                        event_type = event.get("type")
                         
-                        # Handle different message types
-                        if data.get("type") == "response.audio.delta":
-                            # Audio response chunk
+                        # Handle built-in event processing
+                        await self._handle_event(event)
+                        
+                        # Yield specific response types
+                        if event_type == RealtimeEventType.RESPONSE_AUDIO_DELTA.value:
+                            audio_data = event.get("delta", "")
                             yield {
-                                "type": "audio",
-                                "data": data.get("delta", ""),
-                                "format": "pcm16"
+                                "type": "audio_delta",
+                                "data": audio_data,
+                                "format": "pcm16",
+                                "raw_event": event
                             }
-                        elif data.get("type") == "response.text.delta":
-                            # Text response chunk
+                        elif event_type == RealtimeEventType.RESPONSE_TEXT_DELTA.value:
+                            text_data = event.get("delta", "")
                             yield {
-                                "type": "text",
-                                "data": data.get("delta", "")
+                                "type": "text_delta",
+                                "data": text_data,
+                                "raw_event": event
                             }
-                        elif data.get("type") == "response.done":
+                        elif event_type == RealtimeEventType.RESPONSE_AUDIO_TRANSCRIPT_DELTA.value:
+                            transcript_data = event.get("delta", "")
+                            yield {
+                                "type": "transcript_delta",
+                                "data": transcript_data,
+                                "raw_event": event
+                            }
+                        elif event_type == RealtimeEventType.RESPONSE_DONE.value:
                             # Response completed
-                            usage = data.get("response", {}).get("usage", {})
+                            response = event.get("response", {})
+                            usage = response.get("usage", {})
                             
                             # Track usage for billing
-                            self._track_usage(
-                                service_type=ServiceType.AUDIO_STT,
+                            await self._track_usage(
+                                service_type=ServiceType.AUDIO_REALTIME,
                                 operation="realtime_response",
                                 input_tokens=usage.get("input_tokens", 0),
                                 output_tokens=usage.get("output_tokens", 0),
                                 metadata={
-                                    "response_id": data.get("response", {}).get("id"),
-                                    "model": self.model_name
+                                    "response_id": response.get("id"),
+                                    "model": self.model_name,
+                                    "status": response.get("status")
                                 }
                             )
                             
                             yield {
-                                "type": "done",
-                                "usage": usage
+                                "type": "response_done",
+                                "response": response,
+                                "usage": usage,
+                                "raw_event": event
+                            }
+                        elif event_type == RealtimeEventType.ERROR.value:
+                            logger.error(f"Realtime API error: {event}")
+                            yield {
+                                "type": "error",
+                                "error": event.get("error", {}),
+                                "raw_event": event
                             }
                         
                         # Call custom message handler if provided
                         if message_handler:
-                            await message_handler(data)
+                            await message_handler(event)
                             
                     except json.JSONDecodeError as e:
                         logger.error(f"Error parsing WebSocket message: {e}")
                         continue
                         
                 elif msg.type == aiohttp.WSMsgType.ERROR:
-                    logger.error(f"WebSocket error: {ws.exception()}")
+                    logger.error(f"WebSocket error: {self.websocket.exception()}")
+                    yield {
+                        "type": "websocket_error",
+                        "error": str(self.websocket.exception())
+                    }
+                    break
+                elif msg.type == aiohttp.WSMsgType.CLOSED:
+                    logger.info("WebSocket connection closed")
+                    self.is_connected = False
                     break
                     
         except Exception as e:
@@ -239,7 +315,8 @@ class OpenAIRealtimeService(BaseService):
         self, 
         audio_data: bytes, 
         instructions: str = "You are a helpful assistant. Respond in audio.",
-        voice: str = "alloy"
+        voice: str = "alloy",
+        **kwargs
     ) -> Dict[str, Any]:
         """Simple audio chat - send audio, get audio response"""
         try:
@@ -252,34 +329,42 @@ class OpenAIRealtimeService(BaseService):
             session_id = session["id"]
             
             # Connect to WebSocket
-            ws = await self.connect_websocket(session_id)
+            await self.connect_websocket()
             
             try:
                 # Send audio
-                await self.send_audio_message(ws, audio_data)
+                await self.send_audio_message(audio_data)
                 
                 # Collect response
                 audio_chunks = []
+                transcript_chunks = []
                 usage_info = {}
                 
-                async for response in self.listen_for_responses(ws):
-                    if response["type"] == "audio":
+                async for response in self.listen_for_responses():
+                    if response["type"] == "audio_delta":
                         audio_chunks.append(response["data"])
-                    elif response["type"] == "done":
+                    elif response["type"] == "transcript_delta":
+                        transcript_chunks.append(response["data"])
+                    elif response["type"] == "response_done":
                         usage_info = response["usage"]
                         break
+                    elif response["type"] == "error":
+                        raise Exception(f"Realtime API error: {response['error']}")
                 
-                # Combine audio chunks
+                # Combine chunks
                 full_audio = "".join(audio_chunks)
+                full_transcript = "".join(transcript_chunks)
                 
                 return {
                     "audio_response": full_audio,
+                    "transcript": full_transcript,
                     "session_id": session_id,
-                    "usage": usage_info
+                    "usage": usage_info,
+                    "format": "pcm16"
                 }
                 
             finally:
-                await ws.close()
+                await self.disconnect()
                 
         except Exception as e:
             logger.error(f"Error in simple audio chat: {e}")
@@ -289,9 +374,10 @@ class OpenAIRealtimeService(BaseService):
         self, 
         text: str, 
         instructions: str = "You are a helpful assistant.",
-        voice: str = "alloy"
+        voice: str = "alloy",
+        **kwargs
     ) -> Dict[str, Any]:
-        """Simple text chat - send text, get audio response"""
+        """Simple text chat - send text, get audio/text response"""
         try:
             # Create session
             session = await self.create_session(
@@ -302,38 +388,46 @@ class OpenAIRealtimeService(BaseService):
             session_id = session["id"]
             
             # Connect to WebSocket
-            ws = await self.connect_websocket(session_id)
+            await self.connect_websocket()
             
             try:
                 # Send text
-                await self.send_text_message(ws, text)
+                await self.send_text_message(text)
                 
                 # Collect response
                 text_response = ""
                 audio_chunks = []
+                transcript_chunks = []
                 usage_info = {}
                 
-                async for response in self.listen_for_responses(ws):
-                    if response["type"] == "text":
+                async for response in self.listen_for_responses():
+                    if response["type"] == "text_delta":
                         text_response += response["data"]
-                    elif response["type"] == "audio":
+                    elif response["type"] == "audio_delta":
                         audio_chunks.append(response["data"])
-                    elif response["type"] == "done":
+                    elif response["type"] == "transcript_delta":
+                        transcript_chunks.append(response["data"])
+                    elif response["type"] == "response_done":
                         usage_info = response["usage"]
                         break
+                    elif response["type"] == "error":
+                        raise Exception(f"Realtime API error: {response['error']}")
                 
-                # Combine audio chunks
+                # Combine chunks
                 full_audio = "".join(audio_chunks)
+                full_transcript = "".join(transcript_chunks)
                 
                 return {
                     "text_response": text_response,
                     "audio_response": full_audio,
+                    "transcript": full_transcript,
                     "session_id": session_id,
-                    "usage": usage_info
+                    "usage": usage_info,
+                    "format": "pcm16"
                 }
                 
             finally:
-                await ws.close()
+                await self.disconnect()
                 
         except Exception as e:
             logger.error(f"Error in simple text chat: {e}")
@@ -347,7 +441,109 @@ class OpenAIRealtimeService(BaseService):
         """Get list of supported audio formats"""
         return ["pcm16", "g711_ulaw", "g711_alaw"]
     
+    def get_session_limits(self) -> Dict[str, Any]:
+        """Get session limits and constraints"""
+        return self.session_limits.copy()
+    
+    async def update_session(self, **kwargs) -> Dict[str, Any]:
+        """Update session configuration"""
+        try:
+            if not self.is_connected or not self.websocket:
+                raise RuntimeError("WebSocket not connected")
+                
+            # Update session config
+            session_update = {k: v for k, v in kwargs.items() if k in self.default_config}
+            
+            if session_update:
+                await self._send_event({
+                    "type": "session.update",
+                    "session": session_update
+                })
+                
+                # Update local config
+                if hasattr(self, 'session_config'):
+                    self.session_config.update(session_update)
+            
+            return {
+                "status": "updated",
+                "updated_fields": list(session_update.keys())
+            }
+            
+        except Exception as e:
+            logger.error(f"Error updating session: {e}")
+            raise
+    
+    async def disconnect(self):
+        """Disconnect from the realtime session"""
+        try:
+            if self.websocket and not self.websocket.closed:
+                await self.websocket.close()
+            
+            if hasattr(self, 'client_session') and self.client_session:
+                await self.client_session.close()
+                
+            self.is_connected = False
+            self.websocket = None
+            
+            logger.info("Disconnected from realtime session")
+            
+        except Exception as e:
+            logger.error(f"Error disconnecting: {e}")
+    
+    async def _send_event(self, event: Dict[str, Any]):
+        """Send an event to the WebSocket"""
+        if not self.websocket or self.websocket.closed:
+            raise RuntimeError("WebSocket not connected")
+            
+        event_json = json.dumps(event)
+        await self.websocket.send_str(event_json)
+        logger.debug(f"Sent event: {event.get('type')}")
+    
+    def _setup_default_handlers(self):
+        """Setup default event handlers for common events"""
+        
+        async def handle_session_created(event):
+            logger.info(f"Session created: {event.get('session', {}).get('id')}")
+            
+        async def handle_session_updated(event):
+            logger.info(f"Session updated: {event.get('session', {})}")
+            
+        async def handle_input_audio_buffer_committed(event):
+            logger.debug(f"Audio buffer committed: {event.get('item_id', 'unknown')}")
+            
+        async def handle_input_audio_buffer_speech_started(event):
+            logger.debug(f"Speech started: {event.get('audio_start_ms', 0)}ms")
+            
+        async def handle_input_audio_buffer_speech_stopped(event):
+            logger.debug(f"Speech stopped: {event.get('audio_end_ms', 0)}ms")
+            
+        async def handle_conversation_item_created(event):
+            item = event.get('item', {})
+            logger.debug(f"Conversation item created: {item.get('type')} - {item.get('id')}")
+            
+        async def handle_response_created(event):
+            response = event.get('response', {})
+            logger.debug(f"Response created: {response.get('id')}")
+            
+        async def handle_rate_limits_updated(event):
+            limits = event.get('rate_limits', [])
+            logger.debug(f"Rate limits updated: {limits}")
+            
+        async def handle_error(event):
+            error = event.get('error', {})
+            logger.error(f"Realtime API error: {error.get('message')} (Code: {error.get('code')})")
+        
+        # Register default handlers
+        self.add_event_handler(RealtimeEventType.SESSION_CREATED, handle_session_created)
+        self.add_event_handler(RealtimeEventType.SESSION_UPDATED, handle_session_updated)
+        self.add_event_handler(RealtimeEventType.INPUT_AUDIO_BUFFER_COMMITTED, handle_input_audio_buffer_committed)
+        self.add_event_handler(RealtimeEventType.INPUT_AUDIO_BUFFER_SPEECH_STARTED, handle_input_audio_buffer_speech_started)
+        self.add_event_handler(RealtimeEventType.INPUT_AUDIO_BUFFER_SPEECH_STOPPED, handle_input_audio_buffer_speech_stopped)
+        self.add_event_handler(RealtimeEventType.CONVERSATION_ITEM_CREATED, handle_conversation_item_created)
+        self.add_event_handler(RealtimeEventType.RESPONSE_CREATED, handle_response_created)
+        self.add_event_handler(RealtimeEventType.RATE_LIMITS_UPDATED, handle_rate_limits_updated)
+        self.add_event_handler(RealtimeEventType.ERROR, handle_error)
+    
     async def close(self):
         """Cleanup resources"""
-        # No persistent connections to close for REST API
-        pass 
+        await self.disconnect()