isa-model 0.3.91__py3-none-any.whl → 0.4.3__py3-none-any.whl

isa_model/inference/services/audio/__init__.py

@@ -0,0 +1,21 @@
+"""
+Audio Services - Speech, TTS, and Audio Processing Services
+"""
+
+from .base_stt_service import BaseSTTService
+from .base_tts_service import BaseTTSService
+from .base_realtime_service import BaseRealtimeService
+from .openai_stt_service import OpenAISTTService
+from .openai_tts_service import OpenAITTSService
+from .openai_realtime_service import OpenAIRealtimeService
+from .replicate_tts_service import ReplicateTTSService
+
+__all__ = [
+    'BaseSTTService',
+    'BaseeTTSService', 
+    'BaseRealtimeService',
+    'OpenAISTTService',
+    'OpenAITTSService',
+    'OpenAIRealtimeService',
+    'ReplicateTTSService'
+]

isa_model/inference/services/audio/base_realtime_service.py

@@ -0,0 +1,225 @@
+from abc import ABC, abstractmethod
+from typing import Dict, Any, List, Union, Optional, Callable, AsyncGenerator
+from enum import Enum
+import asyncio
+from isa_model.inference.services.base_service import BaseService
+
+
+class RealtimeEventType(Enum):
+    """Realtime API event types"""
+    # Session events
+    SESSION_CREATED = "session.created"
+    SESSION_UPDATED = "session.updated"
+    
+    # Input audio events
+    INPUT_AUDIO_BUFFER_APPEND = "input_audio_buffer.append"
+    INPUT_AUDIO_BUFFER_COMMIT = "input_audio_buffer.commit"
+    INPUT_AUDIO_BUFFER_CLEAR = "input_audio_buffer.clear"
+    INPUT_AUDIO_BUFFER_COMMITTED = "input_audio_buffer.committed"
+    INPUT_AUDIO_BUFFER_SPEECH_STARTED = "input_audio_buffer.speech_started"
+    INPUT_AUDIO_BUFFER_SPEECH_STOPPED = "input_audio_buffer.speech_stopped"
+    
+    # Conversation events
+    CONVERSATION_ITEM_CREATE = "conversation.item.create"
+    CONVERSATION_ITEM_CREATED = "conversation.item.created"
+    CONVERSATION_ITEM_DELETE = "conversation.item.delete"
+    CONVERSATION_ITEM_DELETED = "conversation.item.deleted"
+    CONVERSATION_ITEM_TRUNCATE = "conversation.item.truncate"
+    CONVERSATION_ITEM_TRUNCATED = "conversation.item.truncated"
+    
+    # Response events
+    RESPONSE_CREATE = "response.create"
+    RESPONSE_CREATED = "response.created"
+    RESPONSE_DONE = "response.done"
+    RESPONSE_OUTPUT_ITEM_ADDED = "response.output_item.added"
+    RESPONSE_OUTPUT_ITEM_DONE = "response.output_item.done"
+    RESPONSE_CONTENT_PART_ADDED = "response.content_part.added"
+    RESPONSE_CONTENT_PART_DONE = "response.content_part.done"
+    RESPONSE_TEXT_DELTA = "response.text.delta"
+    RESPONSE_TEXT_DONE = "response.text.done"
+    RESPONSE_AUDIO_TRANSCRIPT_DELTA = "response.audio_transcript.delta"
+    RESPONSE_AUDIO_TRANSCRIPT_DONE = "response.audio_transcript.done"
+    RESPONSE_AUDIO_DELTA = "response.audio.delta"
+    RESPONSE_AUDIO_DONE = "response.audio.done"
+    RESPONSE_FUNCTION_CALL_ARGUMENTS_DELTA = "response.function_call_arguments.delta"
+    RESPONSE_FUNCTION_CALL_ARGUMENTS_DONE = "response.function_call_arguments.done"
+    
+    # Rate limit events
+    RATE_LIMITS_UPDATED = "rate_limits.updated"
+    
+    # Error events
+    ERROR = "error"
+
+
+class BaseRealtimeService(BaseService):
+    """Base class for Realtime API services"""
+    
+    def __init__(self, provider_name: str, model_name: str, **kwargs):
+        super().__init__(provider_name, model_name, **kwargs)
+        self.session_id: Optional[str] = None
+        self.websocket = None
+        self.event_handlers: Dict[str, List[Callable]] = {}
+        self.is_connected = False
+        
+    async def invoke(
+        self, 
+        task: str,
+        **kwargs
+    ) -> Union[Dict[str, Any], List[Dict[str, Any]]]:
+        """
+        统一的任务分发方法 - 支持实时对话任务
+        
+        Args:
+            task: 任务类型，支持多种实时对话任务
+            **kwargs: 任务特定的附加参数
+            
+        Returns:
+            Dict containing task results
+        """
+        if task == "create_session":
+            return await self.create_session(**kwargs)
+        elif task == "connect":
+            return await self.connect_websocket(**kwargs)
+        elif task == "send_audio":
+            if not kwargs.get("audio_data"):
+                raise ValueError("audio_data is required for send_audio task")
+            return await self.send_audio_message(kwargs["audio_data"], **kwargs)
+        elif task == "send_text":
+            if not kwargs.get("text"):
+                raise ValueError("text is required for send_text task")
+            return await self.send_text_message(kwargs["text"], **kwargs)
+        elif task == "listen":
+            return await self.listen_for_responses(**kwargs)
+        elif task == "audio_chat":
+            return await self.simple_audio_chat(**kwargs)
+        elif task == "text_chat":
+            return await self.simple_text_chat(**kwargs)
+        else:
+            raise NotImplementedError(f"{self.__class__.__name__} does not support task: {task}")
+    
+    def get_supported_tasks(self) -> List[str]:
+        """获取支持的任务列表"""
+        return [
+            "create_session", "connect", "send_audio", "send_text", 
+            "listen", "audio_chat", "text_chat"
+        ]
+    
+    @abstractmethod
+    async def create_session(
+        self, 
+        instructions: str = "You are a helpful assistant.",
+        modalities: Optional[List[str]] = None,
+        voice: str = "alloy",
+        **kwargs
+    ) -> Dict[str, Any]:
+        """Create a new realtime session"""
+        pass
+    
+    @abstractmethod
+    async def connect_websocket(self, **kwargs) -> bool:
+        """Connect to the realtime WebSocket"""
+        pass
+    
+    @abstractmethod
+    async def send_audio_message(
+        self, 
+        audio_data: bytes,
+        format: str = "pcm16",
+        **kwargs
+    ) -> Dict[str, Any]:
+        """Send audio data to the realtime session"""
+        pass
+    
+    @abstractmethod
+    async def send_text_message(
+        self, 
+        text: str,
+        **kwargs
+    ) -> Dict[str, Any]:
+        """Send text message to the realtime session"""
+        pass
+    
+    @abstractmethod
+    async def listen_for_responses(
+        self,
+        message_handler: Optional[Callable] = None,
+        **kwargs
+    ) -> AsyncGenerator[Dict[str, Any], None]:
+        """Listen for responses from the realtime session"""
+        pass
+    
+    @abstractmethod
+    async def simple_audio_chat(
+        self, 
+        audio_data: bytes, 
+        instructions: str = "You are a helpful assistant. Respond in audio.",
+        voice: str = "alloy",
+        **kwargs
+    ) -> Dict[str, Any]:
+        """Simple audio chat - send audio, get audio response"""
+        pass
+    
+    @abstractmethod
+    async def simple_text_chat(
+        self, 
+        text: str, 
+        instructions: str = "You are a helpful assistant.",
+        voice: str = "alloy",
+        **kwargs
+    ) -> Dict[str, Any]:
+        """Simple text chat - send text, get audio/text response"""
+        pass
+    
+    def add_event_handler(self, event_type: Union[str, RealtimeEventType], handler: Callable):
+        """Add event handler for specific event type"""
+        event_name = event_type.value if isinstance(event_type, RealtimeEventType) else event_type
+        if event_name not in self.event_handlers:
+            self.event_handlers[event_name] = []
+        self.event_handlers[event_name].append(handler)
+    
+    def remove_event_handler(self, event_type: Union[str, RealtimeEventType], handler: Callable):
+        """Remove event handler"""
+        event_name = event_type.value if isinstance(event_type, RealtimeEventType) else event_type
+        if event_name in self.event_handlers:
+            self.event_handlers[event_name].remove(handler)
+    
+    async def _handle_event(self, event: Dict[str, Any]):
+        """Handle incoming events"""
+        event_type = event.get("type")
+        if event_type in self.event_handlers:
+            for handler in self.event_handlers[event_type]:
+                try:
+                    await handler(event)
+                except Exception as e:
+                    import logging
+                    logging.getLogger(__name__).error(f"Error in event handler for {event_type}: {e}")
+    
+    @abstractmethod
+    def get_supported_voices(self) -> List[str]:
+        """Get list of supported voice options"""
+        pass
+    
+    @abstractmethod
+    def get_supported_formats(self) -> List[str]:
+        """Get list of supported audio formats"""
+        pass
+    
+    @abstractmethod
+    def get_session_limits(self) -> Dict[str, Any]:
+        """Get session limits and constraints"""
+        pass
+    
+    @abstractmethod
+    async def update_session(self, **kwargs) -> Dict[str, Any]:
+        """Update session configuration"""
+        pass
+    
+    @abstractmethod
+    async def disconnect(self):
+        """Disconnect from the realtime session"""
+        pass
+    
+    @abstractmethod
+    async def close(self):
+        """Cleanup resources"""
+        pass

isa_model/inference/services/audio/base_stt_service.py

@@ -1,13 +1,172 @@
 from abc import ABC, abstractmethod
 from typing import Dict, Any, List, Union, Optional, BinaryIO
+import aiohttp
+import asyncio
+import tempfile
+import os
+import logging
+from io import BytesIO
 from isa_model.inference.services.base_service import BaseService
 
+logger = logging.getLogger(__name__)
+
 class BaseSTTService(BaseService):
-    """Base class for Speech-to-Text services with unified task dispatch"""
+    """Base class for Speech-to-Text services with unified task dispatch and URL support"""
+    
+    async def _prepare_audio_input(self, audio_input: Union[str, BinaryIO, bytes]) -> Union[str, BinaryIO]:
+        """
+        Prepare audio input by handling URLs, file paths, bytes data, and file objects
+        
+        Args:
+            audio_input: Audio input (URL, file path, bytes data, or file object)
+            
+        Returns:
+            Prepared audio input (local file path or file object)
+        """
+        if isinstance(audio_input, bytes):
+            # Handle bytes data from API uploads
+            logger.info(f"Converting bytes data to temporary file ({len(audio_input)} bytes)")
+            return await self._save_bytes_to_temp_file(audio_input)
+        elif isinstance(audio_input, str):
+            # Check if it's a URL
+            if audio_input.startswith(('http://', 'https://')):
+                logger.info(f"Downloading audio from URL: {audio_input}")
+                return await self._download_audio_url(audio_input)
+            else:
+                # Regular file path or base64 string
+                return audio_input
+        else:
+            # Already a file object
+            return audio_input
+    
+    async def _prepare_audio_input_with_context(self, audio_input: Union[str, BinaryIO, bytes], context: Dict[str, Any]) -> Union[str, BinaryIO]:
+        """
+        Prepare audio input with additional context from kwargs
+        
+        Args:
+            audio_input: Audio input (URL, file path, bytes data, or file object)
+            context: Additional context including filename, content_type
+            
+        Returns:
+            Prepared audio input (local file path or file object)
+        """
+        if isinstance(audio_input, bytes):
+            # Handle bytes data from API uploads
+            filename = context.get('filename')
+            content_type = context.get('content_type')
+            logger.info(f"Converting bytes data to temporary file ({len(audio_input)} bytes), filename={filename}, content_type={content_type}")
+            return await self._save_bytes_to_temp_file(audio_input, filename, content_type)
+        else:
+            return await self._prepare_audio_input(audio_input)
+    
+    async def _download_audio_url(self, url: str) -> str:
+        """
+        Download audio file from URL to temporary file
+        
+        Args:
+            url: HTTP/HTTPS URL to audio file
+            
+        Returns:
+            Path to downloaded temporary file
+            
+        Raises:
+            Exception: If download fails
+        """
+        try:
+            async with aiohttp.ClientSession() as session:
+                async with session.get(url) as response:
+                    if response.status != 200:
+                        raise Exception(f"Failed to download audio: HTTP {response.status}")
+                    
+                    # Get content type to determine file extension
+                    content_type = response.headers.get('Content-Type', '')
+                    file_ext = self._get_file_extension_from_content_type(content_type)
+                    
+                    # Create temporary file
+                    temp_file = tempfile.NamedTemporaryFile(
+                        delete=False, 
+                        suffix=file_ext,
+                        prefix='audio_download_'
+                    )
+                    
+                    # Download and save
+                    async for chunk in response.content.iter_chunked(8192):
+                        temp_file.write(chunk)
+                    
+                    temp_file.close()
+                    logger.info(f"Downloaded audio to temporary file: {temp_file.name}")
+                    return temp_file.name
+                    
+        except Exception as e:
+            logger.error(f"Failed to download audio from URL {url}: {e}")
+            raise Exception(f"Audio URL download failed: {e}") from e
+    
+    def _get_file_extension_from_content_type(self, content_type: str) -> str:
+        """Get appropriate file extension from Content-Type header"""
+        content_type_map = {
+            'audio/mpeg': '.mp3',
+            'audio/mp3': '.mp3',
+            'audio/wav': '.wav',
+            'audio/wave': '.wav',
+            'audio/x-wav': '.wav',
+            'audio/flac': '.flac',
+            'audio/ogg': '.ogg',
+            'audio/m4a': '.m4a',
+            'audio/mp4': '.mp4',
+            'audio/webm': '.webm'
+        }
+        return content_type_map.get(content_type.lower(), '.audio')
+    
+    async def _save_bytes_to_temp_file(self, audio_bytes: bytes, filename: Optional[str] = None, content_type: Optional[str] = None) -> str:
+        """
+        Save audio bytes data to temporary file
+        
+        Args:
+            audio_bytes: Audio data as bytes
+            filename: Optional filename to determine extension
+            content_type: Optional content type to determine extension
+            
+        Returns:
+            Path to temporary file containing audio data
+        """
+        try:
+            # Determine file extension from filename or content type
+            suffix = '.mp3'  # Default
+            if filename and '.' in filename:
+                suffix = '.' + filename.split('.')[-1]
+            elif content_type:
+                suffix = self._get_file_extension_from_content_type(content_type)
+            
+            # Create temporary file with proper audio extension
+            temp_file = tempfile.NamedTemporaryFile(
+                delete=False,
+                suffix=suffix,
+                prefix='audio_bytes_'
+            )
+            
+            # Write bytes data
+            temp_file.write(audio_bytes)
+            temp_file.close()
+            
+            logger.info(f"Saved {len(audio_bytes)} bytes to temporary file: {temp_file.name}")
+            return temp_file.name
+            
+        except Exception as e:
+            logger.error(f"Failed to save audio bytes to temporary file: {e}")
+            raise Exception(f"Audio bytes save failed: {e}") from e
+    
+    def _cleanup_temp_file(self, file_path: str):
+        """Clean up temporary downloaded file"""
+        try:
+            if file_path and file_path.startswith(tempfile.gettempdir()):
+                os.unlink(file_path)
+                logger.debug(f"Cleaned up temporary file: {file_path}")
+        except Exception as e:
+            logger.warning(f"Failed to cleanup temporary file {file_path}: {e}")
     
     async def invoke(
         self, 
-        audio_input: Union[str, BinaryIO, List[Union[str, BinaryIO]]],
+        audio_input: Union[str, BinaryIO, bytes, List[Union[str, BinaryIO, bytes]]],
         task: Optional[str] = None,
         **kwargs
     ) -> Union[Dict[str, Any], List[Dict[str, Any]]]:
@@ -30,33 +189,47 @@ class BaseSTTService(BaseService):
         # ==================== 语音转文本类任务 ====================
         if task == "transcribe":
             if isinstance(audio_input, list):
+                # Prepare all audio inputs (handle URLs)
+                prepared_inputs = []
+                for audio in audio_input:
+                    prepared_input = await self._prepare_audio_input_with_context(audio, kwargs)
+                    prepared_inputs.append(prepared_input)
                 return await self.transcribe_batch(
-                    audio_input,
+                    prepared_inputs,
                     kwargs.get("language"),
                     kwargs.get("prompt")
                 )
             else:
+                # Prepare single audio input (handle URLs)
+                prepared_input = await self._prepare_audio_input_with_context(audio_input, kwargs)
                 return await self.transcribe(
-                    audio_input,
+                    prepared_input,
                     kwargs.get("language"),
                     kwargs.get("prompt")
                 )
         elif task == "translate":
             if isinstance(audio_input, list):
                 raise ValueError("translate task requires single audio input")
-            return await self.translate(audio_input)
+            prepared_input = await self._prepare_audio_input_with_context(audio_input, kwargs)
+            return await self.translate(prepared_input)
         elif task == "batch_transcribe":
             if not isinstance(audio_input, list):
                 audio_input = [audio_input]
+            # Prepare all audio inputs (handle URLs)
+            prepared_inputs = []
+            for audio in audio_input:
+                prepared_input = await self._prepare_audio_input_with_context(audio, kwargs)
+                prepared_inputs.append(prepared_input)
             return await self.transcribe_batch(
-                audio_input,
+                prepared_inputs,
                 kwargs.get("language"),
                 kwargs.get("prompt")
             )
         elif task == "detect_language":
             if isinstance(audio_input, list):
                 raise ValueError("detect_language task requires single audio input")
-            return await self.detect_language(audio_input)
+            prepared_input = await self._prepare_audio_input_with_context(audio_input, kwargs)
+            return await self.detect_language(prepared_input)
         else:
             raise NotImplementedError(f"{self.__class__.__name__} does not support task: {task}")
     
@@ -72,7 +245,7 @@ class BaseSTTService(BaseService):
     @abstractmethod
     async def transcribe(
         self, 
-        audio_file: Union[str, BinaryIO], 
+        audio_file: Union[str, BinaryIO, bytes], 
         language: Optional[str] = None,
         prompt: Optional[str] = None
     ) -> Dict[str, Any]:
@@ -96,7 +269,7 @@ class BaseSTTService(BaseService):
     @abstractmethod
     async def translate(
         self, 
-        audio_file: Union[str, BinaryIO]
+        audio_file: Union[str, BinaryIO, bytes]
     ) -> Dict[str, Any]:
         """
         Translate audio file to English text
@@ -115,7 +288,7 @@ class BaseSTTService(BaseService):
     @abstractmethod
     async def transcribe_batch(
         self, 
-        audio_files: List[Union[str, BinaryIO]], 
+        audio_files: List[Union[str, BinaryIO, bytes]], 
         language: Optional[str] = None,
         prompt: Optional[str] = None
     ) -> List[Dict[str, Any]]:
@@ -133,7 +306,7 @@ class BaseSTTService(BaseService):
         pass
     
     @abstractmethod
-    async def detect_language(self, audio_file: Union[str, BinaryIO]) -> Dict[str, Any]:
+    async def detect_language(self, audio_file: Union[str, BinaryIO, bytes]) -> Dict[str, Any]:
         """
         Detect language of audio file