isa-model 0.3.9__py3-none-any.whl → 0.4.0__py3-none-any.whl

isa_model/inference/ai_factory.py

@@ -110,6 +110,7 @@ class AIFactory:
         
         Args:
             model_name: Model name. Special names:
+                       - "hybrid": Unified UI/Document analysis service (RECOMMENDED)
                        - "isa_vision_table": Table extraction service
                        - "isa_vision_ui": UI detection service  
                        - "isa_vision_doc": Document analysis service
@@ -120,18 +121,27 @@ class AIFactory:
         Returns:
             Vision service instance
         """
-        # Handle special ISA vision services
-        if model_name in ["isa_vision_table", "isa_vision_ui", "isa_vision_doc"]:
+        # Handle special vision services  
+        if model_name == "hybrid":
+            # Hybrid vision service has been deprecated, use OpenAI as fallback
+            logger.warning("HybridVisionService is deprecated, using OpenAI vision service as fallback")
+            final_provider = "openai"
+            final_model_name = "gpt-4.1-nano"
+        
+        elif model_name in ["isa_vision_table", "isa_vision_ui", "isa_vision_doc"]:
             try:
                 from isa_model.deployment.services.simple_auto_deploy_vision_service import SimpleAutoDeployVisionService
                 logger.info(f"Creating auto-deploy service wrapper for {model_name}")
                 return SimpleAutoDeployVisionService(model_name, config)
             except Exception as e:
                 logger.error(f"Failed to create ISA vision service: {e}")
-                raise
+                # Fallback to ISA service
+                logger.warning(f"Auto-deploy service failed, using ISA vision service as fallback")
+                final_provider = "isa"
+                final_model_name = "isa-omniparser-ui-detection"
         
         # Set defaults for regular services
-        if provider == "openai":
+        elif provider == "openai":
             final_model_name = model_name or "gpt-4.1-mini"
             final_provider = provider
         elif provider == "ollama":
@@ -140,6 +150,9 @@ class AIFactory:
         elif provider == "replicate":
             final_model_name = model_name or "meta/llama-2-70b-chat"
             final_provider = provider
+        elif provider == "isa":
+            final_model_name = model_name or "isa-omniparser-ui-detection"
+            final_provider = provider
         else:
             # Default provider selection
             final_provider = provider or "openai"
@@ -147,6 +160,8 @@ class AIFactory:
                 final_model_name = model_name or "gpt-4.1-mini"
             elif final_provider == "ollama":
                 final_model_name = model_name or "llama3.2-vision:latest"
+            elif final_provider == "isa":
+                final_model_name = model_name or "isa-omniparser-ui-detection"
             else:
                 final_model_name = model_name or "gpt-4.1-mini"
         
@@ -160,6 +175,10 @@ class AIFactory:
                 from isa_model.inference.services.vision.replicate_vision_service import ReplicateVisionService
                 return ReplicateVisionService(provider_name=final_provider, model_name=final_model_name,
                                             model_manager=self.model_manager, config_manager=self.config_manager)
+            elif final_provider == "isa":
+                from isa_model.inference.services.vision.isa_vision_service import ISAVisionService
+                logger.info(f"Creating ISA Vision Service with model: {final_model_name}")
+                return ISAVisionService()
             else:
                 raise ValueError(f"Unsupported vision provider: {final_provider}")
         except Exception as e:
@@ -322,17 +341,34 @@ class AIFactory:
                 final_model_name = model_name or "bge-m3"
         
         # Create service using new centralized approach
+        # Create cache key
+        cache_key = f"embed_{final_provider}_{final_model_name}"
+        
+        # Check cache first
+        if cache_key in self._cached_services:
+            logger.debug(f"Using cached embedding service: {cache_key}")
+            return self._cached_services[cache_key]
+        
         try:
             if final_provider == "openai":
                 from isa_model.inference.services.embedding.openai_embed_service import OpenAIEmbedService
-                return OpenAIEmbedService(provider_name=final_provider, model_name=final_model_name,
-                                        model_manager=self.model_manager, config_manager=self.config_manager)
+                service = OpenAIEmbedService(provider_name=final_provider, model_name=final_model_name,
+                                           model_manager=self.model_manager, config_manager=self.config_manager)
             elif final_provider == "ollama":
                 from isa_model.inference.services.embedding.ollama_embed_service import OllamaEmbedService
-                return OllamaEmbedService(provider_name=final_provider, model_name=final_model_name,
-                                        model_manager=self.model_manager, config_manager=self.config_manager)
+                service = OllamaEmbedService(provider_name=final_provider, model_name=final_model_name,
+                                           model_manager=self.model_manager, config_manager=self.config_manager)
+            elif final_provider == "isa":
+                from isa_model.inference.services.embedding.isa_embed_service import ISAEmbedService
+                service = ISAEmbedService()  # ISA service doesn't use model_manager/config_manager yet
             else:
                 raise ValueError(f"Unsupported embedding provider: {final_provider}")
+            
+            # Cache the service
+            self._cached_services[cache_key] = service
+            logger.debug(f"Created and cached embedding service: {cache_key}")
+            return service
+            
         except Exception as e:
             logger.error(f"Failed to create embedding service: {e}")
             raise

isa_model/inference/services/audio/__init__.py

@@ -0,0 +1,21 @@
+"""
+Audio Services - Speech, TTS, and Audio Processing Services
+"""
+
+from .base_stt_service import BaseSTTService
+from .base_tts_service import BaseTTSService
+from .base_realtime_service import BaseRealtimeService
+from .openai_stt_service import OpenAISTTService
+from .openai_tts_service import OpenAITTSService
+from .openai_realtime_service import OpenAIRealtimeService
+from .replicate_tts_service import ReplicateTTSService
+
+__all__ = [
+    'BaseSTTService',
+    'BaseeTTSService', 
+    'BaseRealtimeService',
+    'OpenAISTTService',
+    'OpenAITTSService',
+    'OpenAIRealtimeService',
+    'ReplicateTTSService'
+]

isa_model/inference/services/audio/base_realtime_service.py

@@ -0,0 +1,225 @@
+from abc import ABC, abstractmethod
+from typing import Dict, Any, List, Union, Optional, Callable, AsyncGenerator
+from enum import Enum
+import asyncio
+from isa_model.inference.services.base_service import BaseService
+
+
+class RealtimeEventType(Enum):
+    """Realtime API event types"""
+    # Session events
+    SESSION_CREATED = "session.created"
+    SESSION_UPDATED = "session.updated"
+    
+    # Input audio events
+    INPUT_AUDIO_BUFFER_APPEND = "input_audio_buffer.append"
+    INPUT_AUDIO_BUFFER_COMMIT = "input_audio_buffer.commit"
+    INPUT_AUDIO_BUFFER_CLEAR = "input_audio_buffer.clear"
+    INPUT_AUDIO_BUFFER_COMMITTED = "input_audio_buffer.committed"
+    INPUT_AUDIO_BUFFER_SPEECH_STARTED = "input_audio_buffer.speech_started"
+    INPUT_AUDIO_BUFFER_SPEECH_STOPPED = "input_audio_buffer.speech_stopped"
+    
+    # Conversation events
+    CONVERSATION_ITEM_CREATE = "conversation.item.create"
+    CONVERSATION_ITEM_CREATED = "conversation.item.created"
+    CONVERSATION_ITEM_DELETE = "conversation.item.delete"
+    CONVERSATION_ITEM_DELETED = "conversation.item.deleted"
+    CONVERSATION_ITEM_TRUNCATE = "conversation.item.truncate"
+    CONVERSATION_ITEM_TRUNCATED = "conversation.item.truncated"
+    
+    # Response events
+    RESPONSE_CREATE = "response.create"
+    RESPONSE_CREATED = "response.created"
+    RESPONSE_DONE = "response.done"
+    RESPONSE_OUTPUT_ITEM_ADDED = "response.output_item.added"
+    RESPONSE_OUTPUT_ITEM_DONE = "response.output_item.done"
+    RESPONSE_CONTENT_PART_ADDED = "response.content_part.added"
+    RESPONSE_CONTENT_PART_DONE = "response.content_part.done"
+    RESPONSE_TEXT_DELTA = "response.text.delta"
+    RESPONSE_TEXT_DONE = "response.text.done"
+    RESPONSE_AUDIO_TRANSCRIPT_DELTA = "response.audio_transcript.delta"
+    RESPONSE_AUDIO_TRANSCRIPT_DONE = "response.audio_transcript.done"
+    RESPONSE_AUDIO_DELTA = "response.audio.delta"
+    RESPONSE_AUDIO_DONE = "response.audio.done"
+    RESPONSE_FUNCTION_CALL_ARGUMENTS_DELTA = "response.function_call_arguments.delta"
+    RESPONSE_FUNCTION_CALL_ARGUMENTS_DONE = "response.function_call_arguments.done"
+    
+    # Rate limit events
+    RATE_LIMITS_UPDATED = "rate_limits.updated"
+    
+    # Error events
+    ERROR = "error"
+
+
+class BaseRealtimeService(BaseService):
+    """Base class for Realtime API services"""
+    
+    def __init__(self, provider_name: str, model_name: str, **kwargs):
+        super().__init__(provider_name, model_name, **kwargs)
+        self.session_id: Optional[str] = None
+        self.websocket = None
+        self.event_handlers: Dict[str, List[Callable]] = {}
+        self.is_connected = False
+        
+    async def invoke(
+        self, 
+        task: str,
+        **kwargs
+    ) -> Union[Dict[str, Any], List[Dict[str, Any]]]:
+        """
+        统一的任务分发方法 - 支持实时对话任务
+        
+        Args:
+            task: 任务类型，支持多种实时对话任务
+            **kwargs: 任务特定的附加参数
+            
+        Returns:
+            Dict containing task results
+        """
+        if task == "create_session":
+            return await self.create_session(**kwargs)
+        elif task == "connect":
+            return await self.connect_websocket(**kwargs)
+        elif task == "send_audio":
+            if not kwargs.get("audio_data"):
+                raise ValueError("audio_data is required for send_audio task")
+            return await self.send_audio_message(kwargs["audio_data"], **kwargs)
+        elif task == "send_text":
+            if not kwargs.get("text"):
+                raise ValueError("text is required for send_text task")
+            return await self.send_text_message(kwargs["text"], **kwargs)
+        elif task == "listen":
+            return await self.listen_for_responses(**kwargs)
+        elif task == "audio_chat":
+            return await self.simple_audio_chat(**kwargs)
+        elif task == "text_chat":
+            return await self.simple_text_chat(**kwargs)
+        else:
+            raise NotImplementedError(f"{self.__class__.__name__} does not support task: {task}")
+    
+    def get_supported_tasks(self) -> List[str]:
+        """获取支持的任务列表"""
+        return [
+            "create_session", "connect", "send_audio", "send_text", 
+            "listen", "audio_chat", "text_chat"
+        ]
+    
+    @abstractmethod
+    async def create_session(
+        self, 
+        instructions: str = "You are a helpful assistant.",
+        modalities: Optional[List[str]] = None,
+        voice: str = "alloy",
+        **kwargs
+    ) -> Dict[str, Any]:
+        """Create a new realtime session"""
+        pass
+    
+    @abstractmethod
+    async def connect_websocket(self, **kwargs) -> bool:
+        """Connect to the realtime WebSocket"""
+        pass
+    
+    @abstractmethod
+    async def send_audio_message(
+        self, 
+        audio_data: bytes,
+        format: str = "pcm16",
+        **kwargs
+    ) -> Dict[str, Any]:
+        """Send audio data to the realtime session"""
+        pass
+    
+    @abstractmethod
+    async def send_text_message(
+        self, 
+        text: str,
+        **kwargs
+    ) -> Dict[str, Any]:
+        """Send text message to the realtime session"""
+        pass
+    
+    @abstractmethod
+    async def listen_for_responses(
+        self,
+        message_handler: Optional[Callable] = None,
+        **kwargs
+    ) -> AsyncGenerator[Dict[str, Any], None]:
+        """Listen for responses from the realtime session"""
+        pass
+    
+    @abstractmethod
+    async def simple_audio_chat(
+        self, 
+        audio_data: bytes, 
+        instructions: str = "You are a helpful assistant. Respond in audio.",
+        voice: str = "alloy",
+        **kwargs
+    ) -> Dict[str, Any]:
+        """Simple audio chat - send audio, get audio response"""
+        pass
+    
+    @abstractmethod
+    async def simple_text_chat(
+        self, 
+        text: str, 
+        instructions: str = "You are a helpful assistant.",
+        voice: str = "alloy",
+        **kwargs
+    ) -> Dict[str, Any]:
+        """Simple text chat - send text, get audio/text response"""
+        pass
+    
+    def add_event_handler(self, event_type: Union[str, RealtimeEventType], handler: Callable):
+        """Add event handler for specific event type"""
+        event_name = event_type.value if isinstance(event_type, RealtimeEventType) else event_type
+        if event_name not in self.event_handlers:
+            self.event_handlers[event_name] = []
+        self.event_handlers[event_name].append(handler)
+    
+    def remove_event_handler(self, event_type: Union[str, RealtimeEventType], handler: Callable):
+        """Remove event handler"""
+        event_name = event_type.value if isinstance(event_type, RealtimeEventType) else event_type
+        if event_name in self.event_handlers:
+            self.event_handlers[event_name].remove(handler)
+    
+    async def _handle_event(self, event: Dict[str, Any]):
+        """Handle incoming events"""
+        event_type = event.get("type")
+        if event_type in self.event_handlers:
+            for handler in self.event_handlers[event_type]:
+                try:
+                    await handler(event)
+                except Exception as e:
+                    import logging
+                    logging.getLogger(__name__).error(f"Error in event handler for {event_type}: {e}")
+    
+    @abstractmethod
+    def get_supported_voices(self) -> List[str]:
+        """Get list of supported voice options"""
+        pass
+    
+    @abstractmethod
+    def get_supported_formats(self) -> List[str]:
+        """Get list of supported audio formats"""
+        pass
+    
+    @abstractmethod
+    def get_session_limits(self) -> Dict[str, Any]:
+        """Get session limits and constraints"""
+        pass
+    
+    @abstractmethod
+    async def update_session(self, **kwargs) -> Dict[str, Any]:
+        """Update session configuration"""
+        pass
+    
+    @abstractmethod
+    async def disconnect(self):
+        """Disconnect from the realtime session"""
+        pass
+    
+    @abstractmethod
+    async def close(self):
+        """Cleanup resources"""
+        pass