isa-model 0.3.5__py3-none-any.whl → 0.3.7__py3-none-any.whl

isa_model/inference/services/vision/helpers/base_stacked_service.py

@@ -0,0 +1,274 @@
+"""
+Base Stacked Service for orchestrating multiple AI models
+"""
+
+from abc import ABC, abstractmethod
+from typing import Dict, Any, List, Optional, Union, Callable
+import time
+import asyncio
+import logging
+from dataclasses import dataclass
+from enum import Enum
+
+# Import shared types from helpers
+try:
+    from ..helpers.stacked_config import StackedLayerType as LayerType, LayerConfig, LayerResult
+except ImportError:
+    # Fallback definitions if shared config is not available
+    class LayerType(Enum):
+        """Types of processing layers"""
+        INTELLIGENCE = "intelligence"
+        DETECTION = "detection"
+        CLASSIFICATION = "classification"
+        VALIDATION = "validation"
+        TRANSFORMATION = "transformation"
+        GENERATION = "generation"
+        ENHANCEMENT = "enhancement"
+        CONTROL = "control"
+        UPSCALING = "upscaling"
+    
+    @dataclass
+    class LayerConfig:
+        """Configuration for a processing layer"""
+        name: str
+        layer_type: LayerType
+        service_type: str
+        model_name: str
+        parameters: Dict[str, Any]
+        depends_on: List[str]
+        timeout: float = 30.0
+        retry_count: int = 1
+        fallback_enabled: bool = True
+    
+    @dataclass
+    class LayerResult:
+        """Result from a processing layer"""
+        layer_name: str
+        success: bool
+        data: Any
+        metadata: Dict[str, Any]
+        execution_time: float
+        error: Optional[str] = None
+
+logger = logging.getLogger(__name__)
+
+class BaseStackedService(ABC):
+    """
+    Base class for stacked services that orchestrate multiple AI models
+    """
+    
+    def __init__(self, ai_factory, service_name: str):
+        self.ai_factory = ai_factory
+        self.service_name = service_name
+        self.layers: List[LayerConfig] = []
+        self.services: Dict[str, Any] = {}
+        self.results: Dict[str, LayerResult] = {}
+        
+    def add_layer(self, config: LayerConfig):
+        """Add a processing layer to the stack"""
+        self.layers.append(config)
+        logger.info(f"Added layer {config.name} ({config.layer_type.value}) to {self.service_name}")
+    
+    async def initialize_services(self):
+        """Initialize all required services"""
+        for layer in self.layers:
+            service_key = f"{layer.service_type}_{layer.model_name}"
+            
+            if service_key not in self.services:
+                if layer.service_type == 'vision':
+                    if layer.model_name == "default":
+                        # 使用默认vision服务
+                        service = self.ai_factory.get_vision()
+                    elif layer.model_name == "omniparser":
+                        # 使用replicate omniparser
+                        service = self.ai_factory.get_vision(model_name="omniparser", provider="replicate")
+                    else:
+                        # 其他指定模型
+                        service = self.ai_factory.get_vision(model_name=layer.model_name)
+                elif layer.service_type == 'llm':
+                    if layer.model_name == "default":
+                        service = self.ai_factory.get_llm()
+                    else:
+                        service = self.ai_factory.get_llm(model_name=layer.model_name)
+                elif layer.service_type == 'image_gen':
+                    if layer.model_name == "default":
+                        service = self.ai_factory.get_image_gen()
+                    else:
+                        service = self.ai_factory.get_image_gen(model_name=layer.model_name)
+                else:
+                    raise ValueError(f"Unsupported service type: {layer.service_type}")
+                
+                self.services[service_key] = service
+                logger.info(f"Initialized {service_key} service")
+    
+    async def execute_layer(self, layer: LayerConfig, context: Dict[str, Any]) -> LayerResult:
+        """Execute a single layer"""
+        start_time = time.time()
+        
+        try:
+            # Check dependencies
+            for dep in layer.depends_on:
+                if dep not in self.results or not self.results[dep].success:
+                    raise ValueError(f"Dependency {dep} failed or not executed")
+            
+            # Get the service
+            service_key = f"{layer.service_type}_{layer.model_name}"
+            service = self.services[service_key]
+            
+            # Execute layer with timeout
+            data = await asyncio.wait_for(
+                self.execute_layer_logic(layer, service, context),
+                timeout=layer.timeout
+            )
+            
+            execution_time = time.time() - start_time
+            
+            result = LayerResult(
+                layer_name=layer.name,
+                success=True,
+                data=data,
+                metadata={
+                    "layer_type": layer.layer_type.value,
+                    "model": layer.model_name,
+                    "parameters": layer.parameters
+                },
+                execution_time=execution_time
+            )
+            
+            logger.info(f"Layer {layer.name} completed in {execution_time:.2f}s")
+            return result
+            
+        except Exception as e:
+            execution_time = time.time() - start_time
+            error_msg = str(e)
+            
+            logger.error(f"Layer {layer.name} failed after {execution_time:.2f}s: {error_msg}")
+            
+            result = LayerResult(
+                layer_name=layer.name,
+                success=False,
+                data=None,
+                metadata={
+                    "layer_type": layer.layer_type.value,
+                    "model": layer.model_name,
+                    "parameters": layer.parameters
+                },
+                execution_time=execution_time,
+                error=error_msg
+            )
+            
+            # Try fallback if enabled
+            if layer.fallback_enabled:
+                fallback_result = await self.execute_fallback(layer, context, error_msg)
+                if fallback_result:
+                    result.data = fallback_result
+                    result.success = True
+                    result.error = f"Fallback used: {error_msg}"
+            
+            return result
+    
+    @abstractmethod
+    async def execute_layer_logic(self, layer: LayerConfig, service: Any, context: Dict[str, Any]) -> Any:
+        """Execute the specific logic for a layer - to be implemented by subclasses"""
+        pass
+    
+    async def execute_fallback(self, layer: LayerConfig, context: Dict[str, Any], error: str) -> Optional[Any]:
+        """Execute fallback logic for a failed layer - can be overridden by subclasses"""
+        return None
+    
+    async def invoke(self, input_data: Dict[str, Any]) -> Dict[str, Any]:
+        """Invoke the entire stack of layers"""
+        logger.info(f"Starting {self.service_name} stack invocation")
+        stack_start_time = time.time()
+        
+        # Initialize services if not done
+        if not self.services:
+            await self.initialize_services()
+        
+        # Clear previous results
+        self.results.clear()
+        
+        # Build execution order based on dependencies
+        execution_order = self._build_execution_order()
+        
+        # Execute layers in order
+        context = {"input": input_data, "results": self.results}
+        
+        for layer in execution_order:
+            result = await self.execute_layer(layer, context)
+            self.results[layer.name] = result
+            
+            # Update context with result
+            context["results"] = self.results
+            
+            # Stop if critical layer fails
+            if not result.success and not layer.fallback_enabled:
+                logger.error(f"Critical layer {layer.name} failed, stopping execution")
+                break
+        
+        total_time = time.time() - stack_start_time
+        
+        # Generate final result
+        final_result = {
+            "service": self.service_name,
+            "success": all(r.success for r in self.results.values()),
+            "total_execution_time": total_time,
+            "layer_results": {name: result for name, result in self.results.items()},
+            "final_output": self.generate_final_output(self.results)
+        }
+        
+        logger.info(f"{self.service_name} stack invocation completed in {total_time:.2f}s")
+        return final_result
+    
+    def _build_execution_order(self) -> List[LayerConfig]:
+        """Build execution order based on dependencies"""
+        # Simple topological sort
+        ordered = []
+        remaining = self.layers.copy()
+        
+        while remaining:
+            # Find layers with no unmet dependencies
+            ready = []
+            for layer in remaining:
+                deps_met = all(dep in [l.name for l in ordered] for dep in layer.depends_on)
+                if deps_met:
+                    ready.append(layer)
+            
+            if not ready:
+                raise ValueError("Circular dependency detected in layer configuration")
+            
+            # Add ready layers to order
+            ordered.extend(ready)
+            for layer in ready:
+                remaining.remove(layer)
+        
+        return ordered
+    
+    @abstractmethod
+    def generate_final_output(self, results: Dict[str, LayerResult]) -> Any:
+        """Generate final output from all layer results - to be implemented by subclasses"""
+        pass
+    
+    async def close(self):
+        """Close all services"""
+        for service in self.services.values():
+            if hasattr(service, 'close'):
+                await service.close()
+        self.services.clear()
+        logger.info(f"Closed all services for {self.service_name}")
+    
+    def get_performance_metrics(self) -> Dict[str, Any]:
+        """Get performance metrics for the stack"""
+        if not self.results:
+            return {}
+        
+        metrics = {
+            "total_layers": len(self.results),
+            "successful_layers": sum(1 for r in self.results.values() if r.success),
+            "failed_layers": sum(1 for r in self.results.values() if not r.success),
+            "total_execution_time": sum(r.execution_time for r in self.results.values()),
+            "layer_times": {name: r.execution_time for name, r in self.results.items()},
+            "layer_success": {name: r.success for name, r in self.results.items()}
+        }
+        
+        return metrics

isa_model/inference/services/vision/helpers/image_utils.py

@@ -1,12 +1,56 @@
 from io import BytesIO
 from PIL import Image
-from typing import Union
+from typing import Union, BinaryIO, Tuple
 import base64
-# from app.config.config_manager import config_manager  # Commented out to fix import
+import requests
+import os
 import logging
 
 logger = logging.getLogger(__name__)
 
+def get_image_data(image: Union[str, BinaryIO]) -> bytes:
+    """
+    从各种输入类型获取图像数据 (统一的图像数据获取函数)
+    
+    Args:
+        image: 图像路径、URL或二进制数据
+        
+    Returns:
+        bytes: 原始图像数据
+    """
+    try:
+        if isinstance(image, str):
+            if image.startswith(('http://', 'https://')):
+                # 从URL下载
+                response = requests.get(image)
+                response.raise_for_status()
+                return response.content
+            elif image.startswith('data:'):
+                # Data URL格式 (如 data:image/png;base64,...)
+                import base64
+                if 'base64,' in image:
+                    base64_data = image.split('base64,')[1]
+                    return base64.b64decode(base64_data)
+                else:
+                    raise ValueError("Unsupported data URL format")
+            else:
+                # 本地文件路径
+                with open(image, 'rb') as f:
+                    return f.read()
+        elif hasattr(image, 'read'):
+            # 文件类对象
+            data = image.read()
+            if isinstance(data, bytes):
+                return data
+            else:
+                raise ValueError("File-like object did not return bytes")
+        else:
+            # 假设是bytes
+            return bytes(image) if not isinstance(image, bytes) else image
+    except Exception as e:
+        logger.error(f"Error getting image data: {e}")
+        raise
+
 def compress_image(image_data: Union[bytes, BytesIO], max_size: int = 1024) -> bytes:
     """压缩图片以减小大小
     
@@ -56,4 +100,229 @@ def encode_image_to_base64(image_data: bytes) -> str:
         return base64.b64encode(image_data).decode('utf-8')
     except Exception as e:
         logger.error(f"Error encoding image to base64: {e}")
-        raise 
+        raise 
+
+def prepare_image_base64(image: Union[str, BinaryIO], compress: bool = False, max_size: int = 1024) -> str:
+    """
+    将图像准备为base64格式 (统一的base64编码函数)
+    
+    Args:
+        image: 图像输入
+        compress: 是否压缩图像
+        max_size: 压缩时的最大尺寸
+        
+    Returns:
+        str: Base64编码的图像字符串
+    """
+    try:
+        image_data = get_image_data(image)
+        
+        if compress:
+            image_data = compress_image(image_data, max_size)
+        
+        return encode_image_to_base64(image_data)
+    except Exception as e:
+        logger.error(f"Error preparing image base64: {e}")
+        raise
+
+def prepare_image_data_url(image: Union[str, BinaryIO], compress: bool = False, max_size: int = 1024) -> str:
+    """
+    将图像准备为data URL格式 (统一的data URL生成函数)
+    
+    Args:
+        image: 图像输入
+        compress: 是否压缩图像
+        max_size: 压缩时的最大尺寸
+        
+    Returns:
+        str: data URL格式的图像字符串
+    """
+    try:
+        base64_data = prepare_image_base64(image, compress, max_size)
+        mime_type = get_image_mime_type(image)
+        return f"data:{mime_type};base64,{base64_data}"
+    except Exception as e:
+        logger.error(f"Error preparing image data URL: {e}")
+        raise
+
+def get_image_mime_type(image: Union[str, BinaryIO]) -> str:
+    """
+    获取图像的MIME类型 (统一的MIME类型检测函数)
+    
+    Args:
+        image: 图像输入
+        
+    Returns:
+        str: MIME类型
+    """
+    try:
+        if isinstance(image, str):
+            # 文件路径 - 检查扩展名
+            ext = os.path.splitext(image)[1].lower()
+            mime_mapping = {
+                '.jpg': 'image/jpeg',
+                '.jpeg': 'image/jpeg',
+                '.png': 'image/png',
+                '.gif': 'image/gif',
+                '.webp': 'image/webp',
+                '.bmp': 'image/bmp',
+                '.tiff': 'image/tiff'
+            }
+            return mime_mapping.get(ext, 'image/jpeg')
+        else:
+            # 尝试从图像数据检测
+            image_data = get_image_data(image)
+            img = Image.open(BytesIO(image_data))
+            format_mapping = {
+                'JPEG': 'image/jpeg',
+                'PNG': 'image/png',
+                'GIF': 'image/gif',
+                'WEBP': 'image/webp',
+                'BMP': 'image/bmp',
+                'TIFF': 'image/tiff'
+            }
+            return format_mapping.get(img.format, 'image/jpeg')
+    except Exception:
+        # 默认回退
+        return 'image/jpeg'
+
+def get_image_dimensions(image: Union[str, BinaryIO]) -> Tuple[int, int]:
+    """
+    获取图像尺寸 (统一的尺寸获取函数)
+    
+    Args:
+        image: 图像输入
+        
+    Returns:
+        tuple: (width, height)
+    """
+    try:
+        image_data = get_image_data(image)
+        img = Image.open(BytesIO(image_data))
+        return img.size
+    except Exception as e:
+        logger.error(f"Error getting image dimensions: {e}")
+        return (0, 0)
+
+def validate_image_format(image: Union[str, BinaryIO], supported_formats: list = None) -> bool:
+    """
+    验证图像格式是否受支持 (统一的格式验证函数)
+    
+    Args:
+        image: 图像输入
+        supported_formats: 支持的格式列表，默认为常见格式
+        
+    Returns:
+        bool: 如果支持则为True
+    """
+    if supported_formats is None:
+        supported_formats = ['jpg', 'jpeg', 'png', 'gif', 'webp', 'bmp', 'tiff']
+    
+    try:
+        if isinstance(image, str):
+            ext = os.path.splitext(image)[1].lower().lstrip('.')
+            return ext in supported_formats
+        else:
+            # 检查实际图像格式
+            image_data = get_image_data(image)
+            img = Image.open(BytesIO(image_data))
+            return img.format.lower() in [fmt.upper() for fmt in supported_formats]
+    except Exception as e:
+        logger.warning(f"Could not validate image format: {e}")
+        return True  # 默认允许
+
+def parse_coordinates_from_text(text: str) -> list:
+    """
+    从文本响应中解析对象坐标 (统一的解析逻辑)
+    
+    Args:
+        text: 包含坐标信息的文本响应
+        
+    Returns:
+        list: 解析出的对象列表，每个对象包含label, confidence, coordinates, description
+    """
+    objects = []
+    lines = text.split('\n')
+    
+    for line in lines:
+        line = line.strip()
+        if line and ':' in line and ('x=' in line or 'width=' in line):
+            try:
+                # 提取对象名称和详细信息
+                parts = line.split(':', 1)
+                if len(parts) == 2:
+                    object_name = parts[0].strip()
+                    details = parts[1].strip()
+                    
+                    # 使用类似正则表达式的解析提取坐标
+                    coords = {}
+                    for param in ['x', 'y', 'width', 'height']:
+                        param_pattern = f"{param}="
+                        if param_pattern in details:
+                            start_idx = details.find(param_pattern) + len(param_pattern)
+                            end_idx = details.find('%', start_idx)
+                            if end_idx > start_idx:
+                                try:
+                                    value = float(details[start_idx:end_idx])
+                                    coords[param] = value
+                                except ValueError:
+                                    continue
+                    
+                    # 提取描述（坐标之后）
+                    desc_start = details.find(' - ')
+                    description = details[desc_start + 3:] if desc_start != -1 else details
+                    
+                    objects.append({
+                        "label": object_name,
+                        "confidence": 1.0,
+                        "coordinates": coords,
+                        "description": description
+                    })
+                    
+            except Exception:
+                # 对于不匹配预期格式的对象的回退
+                objects.append({
+                    "label": line,
+                    "confidence": 1.0,
+                    "coordinates": {},
+                    "description": line
+                })
+    
+    return objects
+
+def parse_center_coordinates_from_text(text: str) -> tuple:
+    """
+    从结构化文本响应中解析中心坐标 (统一的解析逻辑)
+    
+    Args:
+        text: 包含FOUND/CENTER/DESCRIPTION格式的文本响应
+        
+    Returns:
+        tuple: (found: bool, center_coords: List[int] | None, description: str)
+    """
+    found = False
+    center_coords = None
+    description = ""
+    
+    lines = text.split('\n')
+    for line in lines:
+        line = line.strip()
+        if line.startswith('FOUND:'):
+            found = 'YES' in line.upper()
+        elif line.startswith('CENTER:') and found:
+            # 提取中心坐标 [x, y]
+            coords_text = line.replace('CENTER:', '').strip()
+            try:
+                # 移除括号并分割
+                coords_text = coords_text.replace('[', '').replace(']', '')
+                if ',' in coords_text:
+                    x_str, y_str = coords_text.split(',')
+                    x = int(float(x_str.strip()))
+                    y = int(float(y_str.strip()))
+                    center_coords = [x, y]
+            except (ValueError, IndexError):
+                pass
+        elif line.startswith('DESCRIPTION:'):
+            description = line.replace('DESCRIPTION:', '').strip()
+    
+    return found, center_coords, description