isa-model 0.3.5__py3-none-any.whl → 0.3.6__py3-none-any.whl

isa_model/inference/services/vision/base_vision_service.py

@@ -1,11 +1,18 @@
 from abc import ABC, abstractmethod
 from typing import Dict, Any, List, Union, Optional, BinaryIO
+import logging
+
 from isa_model.inference.services.base_service import BaseService
+from isa_model.inference.services.vision.helpers.image_utils import (
+    get_image_data, prepare_image_base64, prepare_image_data_url, 
+    get_image_mime_type, get_image_dimensions, validate_image_format
+)
+
+logger = logging.getLogger(__name__)
 
 class BaseVisionService(BaseService):
-    """Base class for vision understanding services"""
+    """Base class for vision understanding services with common task implementations"""
     
-    @abstractmethod
     async def invoke(
         self, 
         image: Union[str, BinaryIO],
@@ -14,20 +21,46 @@ class BaseVisionService(BaseService):
         **kwargs
     ) -> Dict[str, Any]:
         """
-        Unified invoke method for all vision operations
+        统一的任务分发方法 - Base类提供通用实现
         
         Args:
             image: Path to image file or image data
             prompt: Optional text prompt/question about the image
-            task: Task type (analyze, describe, extract_text, detect_objects, etc.)
+            task: Task type - 支持两大类：图像理解 + 检测抽取
             **kwargs: Additional task-specific parameters
             
         Returns:
             Dict containing task results
         """
-        pass
+        task = task or "analyze"
+        
+        # ==================== 图像理解类任务 ====================
+        if task == "analyze":
+            return await self.analyze_image(image, prompt, kwargs.get("max_tokens", 1000))
+        elif task == "describe":
+            return await self.describe_image(image, kwargs.get("detail_level", "medium"))
+        elif task == "classify":
+            return await self.classify_image(image, kwargs.get("categories"))
+        elif task == "compare":
+            return await self.compare_images(image, kwargs.get("image2"))
+        
+        # ==================== 检测抽取类任务 ====================
+        elif task == "extract_text":
+            return await self.extract_text(image)
+        elif task == "detect_objects":
+            return await self.detect_objects(image, kwargs.get("confidence_threshold", 0.5))
+        elif task == "detect_ui_elements":
+            return await self.detect_ui_elements(image, kwargs.get("element_types"), kwargs.get("confidence_threshold", 0.5))
+        elif task == "detect_document_elements":
+            return await self.detect_document_elements(image, kwargs.get("element_types"), kwargs.get("confidence_threshold", 0.5))
+        elif task == "extract_table_data":
+            return await self.extract_table_data(image, kwargs.get("table_format", "json"), kwargs.get("preserve_formatting", True))
+        elif task == "get_coordinates":
+            return await self.get_object_coordinates(image, kwargs.get("object_name", ""))
+        
+        else:
+            raise NotImplementedError(f"{self.__class__.__name__} does not support task: {task}")
     
-    @abstractmethod
     async def analyze_image(
         self, 
         image: Union[str, BinaryIO],
@@ -35,7 +68,7 @@ class BaseVisionService(BaseService):
         max_tokens: int = 1000
     ) -> Dict[str, Any]:
         """
-        Analyze image and provide description or answer questions
+        通用图像分析 - Provider可选实现
         
         Args:
             image: Path to image file or image data
@@ -49,173 +82,241 @@ class BaseVisionService(BaseService):
             - detected_objects: List of detected objects (if available)
             - metadata: Additional metadata about the analysis
         """
-        pass
+        raise NotImplementedError(f"{self.__class__.__name__} does not support analyze_image task")
     
-    @abstractmethod
-    async def analyze_images(
+    # ==================== 图像理解类方法 ====================
+    
+    async def describe_image(
         self, 
-        images: List[Union[str, BinaryIO]],
-        prompt: Optional[str] = None,
-        max_tokens: int = 1000
-    ) -> List[Dict[str, Any]]:
+        image: Union[str, BinaryIO],
+        detail_level: str = "medium"
+    ) -> Dict[str, Any]:
         """
-        Analyze multiple images
-        
-        Args:
-            images: List of image paths or image data
-            prompt: Optional text prompt/question about the images
-            max_tokens: Maximum tokens in response
-            
-        Returns:
-            List of analysis result dictionaries
+        图像描述 - Provider可选实现
         """
-        pass
+        raise NotImplementedError(f"{self.__class__.__name__} does not support describe_image task")
     
-    @abstractmethod
-    async def describe_image(
+    async def classify_image(
         self, 
         image: Union[str, BinaryIO],
-        detail_level: str = "medium"
+        categories: Optional[List[str]] = None
     ) -> Dict[str, Any]:
         """
-        Generate detailed description of image
-        
-        Args:
-            image: Path to image file or image data
-            detail_level: Level of detail ("low", "medium", "high")
-            
-        Returns:
-            Dict containing description results with keys:
-            - description: Detailed text description
-            - objects: List of detected objects
-            - scene: Scene description
-            - colors: Dominant colors
+        图像分类 - Provider可选实现
         """
-        pass
+        raise NotImplementedError(f"{self.__class__.__name__} does not support classify_image task")
+    
+    async def compare_images(
+        self, 
+        image1: Union[str, BinaryIO],
+        image2: Union[str, BinaryIO]
+    ) -> Dict[str, Any]:
+        """
+        图像比较 - Provider可选实现
+        """
+        raise NotImplementedError(f"{self.__class__.__name__} does not support compare_images task")
+    
+    # ==================== 检测抽取类方法 ====================
     
-    @abstractmethod
     async def extract_text(self, image: Union[str, BinaryIO]) -> Dict[str, Any]:
         """
-        Extract text from image (OCR)
-        
-        Args:
-            image: Path to image file or image data
-            
-        Returns:
-            Dict containing OCR results with keys:
-            - text: Extracted text
-            - confidence: Overall confidence score
-            - bounding_boxes: Text regions with coordinates (if available)
-            - language: Detected language (if available)
+        文本提取(OCR) - Provider可选实现
         """
-        pass
+        raise NotImplementedError(f"{self.__class__.__name__} does not support extract_text task")
     
-    @abstractmethod
     async def detect_objects(
         self, 
         image: Union[str, BinaryIO],
         confidence_threshold: float = 0.5
     ) -> Dict[str, Any]:
         """
-        Detect objects in image
-        
-        Args:
-            image: Path to image file or image data
-            confidence_threshold: Minimum confidence for detections
-            
-        Returns:
-            Dict containing detection results with keys:
-            - objects: List of detected objects with labels, confidence, and coordinates
-            - count: Number of objects detected
-            - bounding_boxes: Object locations with coordinates
+        通用物体检测 - Provider可选实现
         """
-        pass
+        raise NotImplementedError(f"{self.__class__.__name__} does not support detect_objects task")
     
-    @abstractmethod
-    async def get_object_coordinates(
+    async def detect_ui_elements(
         self,
         image: Union[str, BinaryIO],
-        object_name: str
+        element_types: Optional[List[str]] = None,
+        confidence_threshold: float = 0.5
     ) -> Dict[str, Any]:
         """
-        Get coordinates of a specific object in the image
+        UI界面元素检测 - Provider可选实现
         
         Args:
-            image: Path to image file or image data
-            object_name: Name of the object to locate
+            image: 输入图像
+            element_types: 要检测的元素类型 ['button', 'input', 'text', 'image', 'link', etc.]
+            confidence_threshold: 置信度阈值
             
         Returns:
-            Dict containing coordinate results with keys:
-            - found: Boolean indicating if object was found
-            - center_coordinates: List [x, y] with pixel coordinates of center point
-            - confidence: Confidence score for the detection
-            - description: Description of the object location
+            Dict containing detected UI elements with their bounding boxes and types
         """
-        pass
+        raise NotImplementedError(f"{self.__class__.__name__} does not support detect_ui_elements task")
     
-    @abstractmethod
-    async def classify_image(
-        self, 
+    async def detect_document_elements(
+        self,
         image: Union[str, BinaryIO],
-        categories: Optional[List[str]] = None
+        element_types: Optional[List[str]] = None,
+        confidence_threshold: float = 0.5
     ) -> Dict[str, Any]:
         """
-        Classify image into categories
+        文档结构元素检测 - Provider可选实现
         
         Args:
-            image: Path to image file or image data
-            categories: Optional list of specific categories to consider
+            image: 输入图像
+            element_types: 要检测的元素类型 ['table', 'header', 'paragraph', 'list', etc.]
+            confidence_threshold: 置信度阈值
             
         Returns:
-            Dict containing classification results with keys:
-            - category: Top predicted category
-            - confidence: Confidence score
-            - all_predictions: List of all predictions with scores
+            Dict containing detected document elements with their structure and content
         """
-        pass
+        raise NotImplementedError(f"{self.__class__.__name__} does not support detect_document_elements task")
     
-    @abstractmethod
-    async def compare_images(
-        self, 
-        image1: Union[str, BinaryIO],
-        image2: Union[str, BinaryIO]
+    async def get_object_coordinates(
+        self,
+        image: Union[str, BinaryIO],
+        object_name: str
+    ) -> Dict[str, Any]:
+        """
+        获取对象坐标 - Provider可选实现
+        """
+        raise NotImplementedError(f"{self.__class__.__name__} does not support get_object_coordinates task")
+    
+    async def extract_table_data(
+        self,
+        image: Union[str, BinaryIO],
+        table_format: str = "json",
+        preserve_formatting: bool = True
     ) -> Dict[str, Any]:
         """
-        Compare two images for similarity
+        表格数据结构化抽取 - Provider可选实现
         
         Args:
-            image1: First image path or data
-            image2: Second image path or data
+            image: 输入图像
+            table_format: 输出格式 ('json', 'csv', 'markdown', 'html')
+            preserve_formatting: 是否保持原始格式（合并单元格、样式等）
             
         Returns:
-            Dict containing comparison results with keys:
-            - similarity_score: Numerical similarity score
-            - differences: Description of key differences
-            - common_elements: Description of common elements
+            Dict containing extracted table data in structured format:
+            {
+                "tables": [
+                    {
+                        "table_id": "table_1",
+                        "headers": ["Column1", "Column2", "Column3"],
+                        "rows": [
+                            ["cell1", "cell2", "cell3"],
+                            ["cell4", "cell5", "cell6"]
+                        ],
+                        "metadata": {
+                            "row_count": 2,
+                            "column_count": 3,
+                            "has_headers": true,
+                            "merged_cells": [],
+                            "table_caption": "optional_caption"
+                        }
+                    }
+                ],
+                "raw_data": "original_table_text",
+                "format": "json"
+            }
         """
+        raise NotImplementedError(f"{self.__class__.__name__} does not support extract_table_data task")
+    
+    async def close(self):
+        """Cleanup resources - default implementation does nothing"""
         pass
     
-    @abstractmethod
-    def get_supported_formats(self) -> List[str]:
+    def get_supported_tasks(self) -> List[str]:
         """
-        Get list of supported image formats
+        获取provider支持的任务列表
         
         Returns:
-            List of supported file extensions (e.g., ['jpg', 'png', 'gif'])
+            List of supported task names
         """
-        pass
+        supported = []
+        
+        # 检查哪些方法被实现了
+        if hasattr(self, 'analyze_image') and callable(getattr(self, 'analyze_image')):
+            try:
+                # 尝试调用看是否抛出NotImplementedError
+                import inspect
+                if not 'NotImplementedError' in inspect.getsource(self.analyze_image):
+                    supported.append('analyze')
+            except:
+                pass
+                
+        # 检查各类任务支持情况
+        method_task_map = {
+            # 图像理解类
+            'describe_image': 'describe',
+            'classify_image': 'classify',
+            'compare_images': 'compare',
+            # 检测抽取类
+            'extract_text': 'extract_text', 
+            'detect_objects': 'detect_objects',
+            'detect_ui_elements': 'detect_ui_elements',
+            'detect_document_elements': 'detect_document_elements',
+            'extract_table_data': 'extract_table_data',
+            'get_object_coordinates': 'get_coordinates'
+        }
+        
+        for method_name, task_name in method_task_map.items():
+            if hasattr(self, method_name):
+                # 检查是否是默认实现（基于analyze_image）还是provider自己的实现
+                supported.append(task_name)
+                
+        return supported
+    
+    # ==================== COMMON TASK IMPLEMENTATIONS ====================
+    # 为每个provider提供可选的默认实现，provider可以覆盖这些方法
+    
+    async def analyze_images(
+        self, 
+        images: List[Union[str, BinaryIO]],
+        prompt: Optional[str] = None,
+        max_tokens: int = 1000
+    ) -> List[Dict[str, Any]]:
+        """
+        批量图像分析 - Provider可选实现
+        默认实现：如果provider支持analyze_image，则逐个调用
+        """
+        if hasattr(self, 'analyze_image'):
+            results = []
+            for image in images:
+                try:
+                    result = await self.analyze_image(image, prompt, max_tokens)
+                    results.append(result)
+                except NotImplementedError:
+                    raise NotImplementedError(f"{self.__class__.__name__} does not support analyze_images task")
+            return results
+        else:
+            raise NotImplementedError(f"{self.__class__.__name__} does not support analyze_images task")
+    
+    
+    def get_supported_formats(self) -> List[str]:
+        """
+        获取支持的图像格式 - Provider应该实现
+        """
+        return ['jpg', 'jpeg', 'png', 'gif', 'webp']  # 通用格式
     
-    @abstractmethod
     def get_max_image_size(self) -> Dict[str, int]:
         """
-        Get maximum supported image dimensions
-        
-        Returns:
-            Dict with 'width' and 'height' keys for maximum dimensions
+        获取最大图像尺寸 - Provider应该实现
         """
-        pass
+        return {"width": 2048, "height": 2048, "file_size_mb": 10}  # 通用限制
     
-    @abstractmethod
-    async def close(self):
-        """Cleanup resources"""
-        pass
+    # ==================== UTILITY METHODS ====================
+    
+    def _parse_coordinates_from_text(self, text: str) -> List[Dict[str, Any]]:
+        """
+        从文本响应中解析对象坐标 - 使用统一的解析工具
+        """
+        from isa_model.inference.services.vision.helpers.image_utils import parse_coordinates_from_text
+        return parse_coordinates_from_text(text)
+    
+    def _parse_center_coordinates_from_text(self, text: str) -> tuple[bool, Optional[List[int]], str]:
+        """
+        从结构化文本响应中解析中心坐标 - 使用统一的解析工具
+        """
+        from isa_model.inference.services.vision.helpers.image_utils import parse_center_coordinates_from_text
+        return parse_center_coordinates_from_text(text)

isa_model/inference/services/vision/{isA_vision_service.py → disabled/isA_vision_service.py}

@@ -23,6 +23,7 @@ class ISAVisionService(BaseVisionService):
         super().__init__(provider, model_name)
         self.ui_app = None
         self.doc_app = None
+        self.table_app = None
         self._initialize_modal_connections()
     
     def _initialize_modal_connections(self):
@@ -42,6 +43,14 @@ class ISAVisionService(BaseVisionService):
         except Exception as e:
             logger.warning(f"� Document service not available: {e}")
             self.doc_app = None
+        
+        try:
+            # Connect to table extraction service
+            self.table_app = modal.App.lookup("qwen-vision-table", create_if_missing=False)
+            logger.info("✅ Connected to table extraction service")
+        except Exception as e:
+            logger.warning(f"⚠️ Table extraction service not available: {e}")
+            self.table_app = None
     
     async def invoke(
         self, 
@@ -59,6 +68,8 @@ class ISAVisionService(BaseVisionService):
             return await self.extract_text(image)
         elif task == "analyze_document":
             return await self._analyze_document(image)
+        elif task == "extract_table" or task == "table_extraction":
+            return await self.extract_table_data(image, **kwargs)
         else:
             return await self.analyze_image(image, prompt, **kwargs)
     
@@ -399,4 +410,91 @@ class ISAVisionService(BaseVisionService):
                 'success': False,
                 'error': str(e),
                 'service': 'isa-vision-doc'
+            }
+    
+    async def extract_table_data(
+        self, 
+        image: Union[str, BinaryIO],
+        extraction_format: str = "markdown",
+        custom_prompt: Optional[str] = None
+    ) -> Dict[str, Any]:
+        """Extract table data using Qwen2.5-VL table extraction service"""
+        
+        if not self.table_app:
+            return {
+                'success': False,
+                'error': 'Table extraction service not available',
+                'service': 'isa-vision-table'
+            }
+        
+        try:
+            # Convert image to base64
+            image_b64 = self._encode_image(image)
+            
+            # Call Modal table extraction service
+            table_extractor = modal.Cls.from_name("qwen-vision-table", "QwenTableExtractionService")
+            result = table_extractor().extract_table_data.remote(
+                image_b64=image_b64,
+                extraction_format=extraction_format,
+                custom_prompt=custom_prompt
+            )
+            
+            if result.get('success'):
+                return {
+                    'success': True,
+                    'service': 'isa-vision-table',
+                    'extracted_data': result.get('extracted_data'),
+                    'raw_output': result.get('raw_output'),
+                    'format': result.get('format'),
+                    'processing_time': result.get('processing_time'),
+                    'model_info': result.get('model_info')
+                }
+            else:
+                return {
+                    'success': False,
+                    'error': result.get('error', 'Table extraction failed'),
+                    'service': 'isa-vision-table'
+                }
+                
+        except Exception as e:
+            logger.error(f"Table extraction failed: {e}")
+            return {
+                'success': False,
+                'error': str(e),
+                'service': 'isa-vision-table'
+            }
+    
+    async def batch_extract_tables(
+        self, 
+        images: List[Union[str, BinaryIO]],
+        extraction_format: str = "markdown"
+    ) -> Dict[str, Any]:
+        """Extract tables from multiple images"""
+        
+        if not self.table_app:
+            return {
+                'success': False,
+                'error': 'Table extraction service not available',
+                'service': 'isa-vision-table'
+            }
+        
+        try:
+            # Convert all images to base64
+            images_b64 = [self._encode_image(image) for image in images]
+            
+            # Call Modal batch extraction service
+            table_extractor = modal.Cls.from_name("qwen-vision-table", "QwenTableExtractionService")
+            result = table_extractor().batch_extract_tables.remote(
+                images_b64=images_b64,
+                extraction_format=extraction_format
+            )
+            
+            return result
+            
+        except Exception as e:
+            logger.error(f"Batch table extraction failed: {e}")
+            return {
+                'success': False,
+                'error': str(e),
+                'service': 'isa-vision-table'
             }

isa_model/inference/services/{stacked → vision}/doc_analysis_service.py

@@ -22,7 +22,7 @@ import logging
 from typing import Dict, Any, List, Union, Optional, BinaryIO
 from datetime import datetime
 
-from isa_model.inference.services.stacked.base_stacked_service import (
+from .helpers.base_stacked_service import (
     BaseStackedService, LayerConfig, LayerType, LayerResult
 )