isa-model 0.2.0__py3-none-any.whl → 0.2.9__py3-none-any.whl

isa_model/inference/services/vision/__init__.py

@@ -6,7 +6,7 @@ Vision服务包
 包含所有视觉相关服务模块
 """
 
-# 导出ReplicateVisionService
-from isa_model.inference.services.vision.replicate_image_gen_service import ReplicateVisionService
+# 导出ReplicateImageGenService
+from isa_model.inference.services.vision.replicate_image_gen_service import ReplicateImageGenService
 
-__all__ = ["ReplicateVisionService"] 
+__all__ = ["ReplicateImageGenService"] 

isa_model/inference/services/vision/base_image_gen_service.py

@@ -0,0 +1,161 @@
+from abc import ABC, abstractmethod
+from typing import Dict, Any, List, Union, Optional, BinaryIO
+from isa_model.inference.services.base_service import BaseService
+
+class BaseImageGenService(BaseService):
+    """Base class for image generation services"""
+    
+    @abstractmethod
+    async def generate_image(
+        self, 
+        prompt: str,
+        negative_prompt: Optional[str] = None,
+        width: int = 512,
+        height: int = 512,
+        num_inference_steps: int = 20,
+        guidance_scale: float = 7.5,
+        seed: Optional[int] = None
+    ) -> Dict[str, Any]:
+        """
+        Generate a single image from text prompt
+        
+        Args:
+            prompt: Text description of the desired image
+            negative_prompt: Text describing what to avoid in the image
+            width: Image width in pixels
+            height: Image height in pixels
+            num_inference_steps: Number of denoising steps
+            guidance_scale: How closely to follow the prompt
+            seed: Random seed for reproducible results
+            
+        Returns:
+            Dict containing generation results with keys:
+            - image_data: Binary image data or PIL Image
+            - format: Image format (e.g., 'png', 'jpg')
+            - width: Actual image width
+            - height: Actual image height
+            - seed: Seed used for generation
+        """
+        pass
+    
+    @abstractmethod
+    async def generate_images(
+        self, 
+        prompt: str,
+        num_images: int = 1,
+        negative_prompt: Optional[str] = None,
+        width: int = 512,
+        height: int = 512,
+        num_inference_steps: int = 20,
+        guidance_scale: float = 7.5,
+        seed: Optional[int] = None
+    ) -> List[Dict[str, Any]]:
+        """
+        Generate multiple images from text prompt
+        
+        Args:
+            prompt: Text description of the desired image
+            num_images: Number of images to generate
+            negative_prompt: Text describing what to avoid in the image
+            width: Image width in pixels
+            height: Image height in pixels
+            num_inference_steps: Number of denoising steps
+            guidance_scale: How closely to follow the prompt
+            seed: Random seed for reproducible results
+            
+        Returns:
+            List of generation result dictionaries
+        """
+        pass
+    
+    @abstractmethod
+    async def generate_image_to_file(
+        self, 
+        prompt: str,
+        output_path: str,
+        negative_prompt: Optional[str] = None,
+        width: int = 512,
+        height: int = 512,
+        num_inference_steps: int = 20,
+        guidance_scale: float = 7.5,
+        seed: Optional[int] = None
+    ) -> Dict[str, Any]:
+        """
+        Generate image and save directly to file
+        
+        Args:
+            prompt: Text description of the desired image
+            output_path: Path to save the generated image
+            negative_prompt: Text describing what to avoid in the image
+            width: Image width in pixels
+            height: Image height in pixels
+            num_inference_steps: Number of denoising steps
+            guidance_scale: How closely to follow the prompt
+            seed: Random seed for reproducible results
+            
+        Returns:
+            Dict containing generation results with keys:
+            - file_path: Path to saved image
+            - width: Image width
+            - height: Image height
+            - seed: Seed used for generation
+        """
+        pass
+    
+    @abstractmethod
+    async def image_to_image(
+        self,
+        prompt: str,
+        init_image: Union[str, BinaryIO],
+        strength: float = 0.8,
+        negative_prompt: Optional[str] = None,
+        num_inference_steps: int = 20,
+        guidance_scale: float = 7.5,
+        seed: Optional[int] = None
+    ) -> Dict[str, Any]:
+        """
+        Generate image based on existing image and prompt
+        
+        Args:
+            prompt: Text description of desired modifications
+            init_image: Path to initial image or image data
+            strength: How much to transform the initial image (0.0-1.0)
+            negative_prompt: Text describing what to avoid
+            num_inference_steps: Number of denoising steps
+            guidance_scale: How closely to follow the prompt
+            seed: Random seed for reproducible results
+            
+        Returns:
+            Dict containing generation results
+        """
+        pass
+    
+    @abstractmethod
+    def get_supported_sizes(self) -> List[Dict[str, int]]:
+        """
+        Get list of supported image dimensions
+        
+        Returns:
+            List of dictionaries with 'width' and 'height' keys
+        """
+        pass
+    
+    @abstractmethod
+    def get_model_info(self) -> Dict[str, Any]:
+        """
+        Get information about the image generation model
+        
+        Returns:
+            Dict containing model information:
+            - name: Model name
+            - max_width: Maximum supported width
+            - max_height: Maximum supported height
+            - supports_negative_prompt: Whether negative prompts are supported
+            - supports_img2img: Whether image-to-image is supported
+        """
+        pass
+    
+    @abstractmethod
+    async def close(self):
+        """Cleanup resources"""
+        pass

isa_model/inference/services/vision/base_vision_service.py

@@ -0,0 +1,177 @@
+from abc import ABC, abstractmethod
+from typing import Dict, Any, List, Union, Optional, BinaryIO
+from isa_model.inference.services.base_service import BaseService
+
+class BaseVisionService(BaseService):
+    """Base class for vision understanding services"""
+    
+    @abstractmethod
+    async def analyze_image(
+        self, 
+        image: Union[str, BinaryIO],
+        prompt: Optional[str] = None,
+        max_tokens: int = 1000
+    ) -> Dict[str, Any]:
+        """
+        Analyze image and provide description or answer questions
+        
+        Args:
+            image: Path to image file or image data
+            prompt: Optional text prompt/question about the image
+            max_tokens: Maximum tokens in response
+            
+        Returns:
+            Dict containing analysis results with keys:
+            - text: Description or answer about the image
+            - confidence: Confidence score (if available)
+            - detected_objects: List of detected objects (if available)
+            - metadata: Additional metadata about the analysis
+        """
+        pass
+    
+    @abstractmethod
+    async def analyze_images(
+        self, 
+        images: List[Union[str, BinaryIO]],
+        prompt: Optional[str] = None,
+        max_tokens: int = 1000
+    ) -> List[Dict[str, Any]]:
+        """
+        Analyze multiple images
+        
+        Args:
+            images: List of image paths or image data
+            prompt: Optional text prompt/question about the images
+            max_tokens: Maximum tokens in response
+            
+        Returns:
+            List of analysis result dictionaries
+        """
+        pass
+    
+    @abstractmethod
+    async def describe_image(
+        self, 
+        image: Union[str, BinaryIO],
+        detail_level: str = "medium"
+    ) -> Dict[str, Any]:
+        """
+        Generate detailed description of image
+        
+        Args:
+            image: Path to image file or image data
+            detail_level: Level of detail ("low", "medium", "high")
+            
+        Returns:
+            Dict containing description results with keys:
+            - description: Detailed text description
+            - objects: List of detected objects
+            - scene: Scene description
+            - colors: Dominant colors
+        """
+        pass
+    
+    @abstractmethod
+    async def extract_text(self, image: Union[str, BinaryIO]) -> Dict[str, Any]:
+        """
+        Extract text from image (OCR)
+        
+        Args:
+            image: Path to image file or image data
+            
+        Returns:
+            Dict containing OCR results with keys:
+            - text: Extracted text
+            - confidence: Overall confidence score
+            - bounding_boxes: Text regions with coordinates (if available)
+            - language: Detected language (if available)
+        """
+        pass
+    
+    @abstractmethod
+    async def detect_objects(
+        self, 
+        image: Union[str, BinaryIO],
+        confidence_threshold: float = 0.5
+    ) -> Dict[str, Any]:
+        """
+        Detect objects in image
+        
+        Args:
+            image: Path to image file or image data
+            confidence_threshold: Minimum confidence for detections
+            
+        Returns:
+            Dict containing detection results with keys:
+            - objects: List of detected objects with labels and confidence
+            - count: Number of objects detected
+            - bounding_boxes: Object locations (if available)
+        """
+        pass
+    
+    @abstractmethod
+    async def classify_image(
+        self, 
+        image: Union[str, BinaryIO],
+        categories: Optional[List[str]] = None
+    ) -> Dict[str, Any]:
+        """
+        Classify image into categories
+        
+        Args:
+            image: Path to image file or image data
+            categories: Optional list of specific categories to consider
+            
+        Returns:
+            Dict containing classification results with keys:
+            - category: Top predicted category
+            - confidence: Confidence score
+            - all_predictions: List of all predictions with scores
+        """
+        pass
+    
+    @abstractmethod
+    async def compare_images(
+        self, 
+        image1: Union[str, BinaryIO],
+        image2: Union[str, BinaryIO]
+    ) -> Dict[str, Any]:
+        """
+        Compare two images for similarity
+        
+        Args:
+            image1: First image path or data
+            image2: Second image path or data
+            
+        Returns:
+            Dict containing comparison results with keys:
+            - similarity_score: Numerical similarity score
+            - differences: Description of key differences
+            - common_elements: Description of common elements
+        """
+        pass
+    
+    @abstractmethod
+    def get_supported_formats(self) -> List[str]:
+        """
+        Get list of supported image formats
+        
+        Returns:
+            List of supported file extensions (e.g., ['jpg', 'png', 'gif'])
+        """
+        pass
+    
+    @abstractmethod
+    def get_max_image_size(self) -> Dict[str, int]:
+        """
+        Get maximum supported image dimensions
+        
+        Returns:
+            Dict with 'width' and 'height' keys for maximum dimensions
+        """
+        pass
+    
+    @abstractmethod
+    async def close(self):
+        """Cleanup resources"""
+        pass

isa_model/inference/services/vision/ollama_vision_service.py

@@ -2,15 +2,15 @@ import os
 import json
 import base64
 import ollama
-from typing import Dict, Any, Union
+from typing import Dict, Any, Union, List, Optional, BinaryIO
 from tenacity import retry, stop_after_attempt, wait_exponential
-from isa_model.inference.services.base_service import BaseService
+from isa_model.inference.services.vision.base_vision_service import BaseVisionService
 from isa_model.inference.providers.base_provider import BaseProvider
 import logging
 
 logger = logging.getLogger(__name__)
 
-class OllamaVisionService(BaseService):
+class OllamaVisionService(BaseVisionService):
     """Vision model service wrapper for Ollama using base64 encoded images"""
     
     def __init__(self, provider: 'BaseProvider', model_name: str = 'gemma3:4b'):
@@ -18,30 +18,38 @@ class OllamaVisionService(BaseService):
         self.max_tokens = self.config.get('max_tokens', 1000)
         self.temperature = self.config.get('temperature', 0.7)
     
+    def _get_image_data(self, image: Union[str, BinaryIO]) -> bytes:
+        """获取图像数据"""
+        if isinstance(image, str):
+            with open(image, 'rb') as f:
+                return f.read()
+        else:
+            return image.read()
+
     @retry(
         stop=stop_after_attempt(3),
         wait=wait_exponential(multiplier=1, min=4, max=10),
         reraise=True
     )
-    async def analyze_image(self, image_data: Union[bytes, str], query: str) -> str:
-        """分析图片并返回结果
-        
-        Args:
-            image_data: 图片数据，可以是 bytes 或图片路径字符串
-            query: 查询文本
-            
-        Returns:
-            str: 分析结果
+    async def analyze_image(
+        self, 
+        image: Union[str, BinaryIO],
+        prompt: Optional[str] = None,
+        max_tokens: int = 1000
+    ) -> Dict[str, Any]:
+        """
+        Analyze image and provide description or answer questions
         """
         try:
-            # 如果是文件路径，读取文件内容
-            if isinstance(image_data, str):
-                with open(image_data, 'rb') as f:
-                    image_data = f.read()
+            # 获取图像数据
+            image_data = self._get_image_data(image)
             
             # 转换为base64
             image_base64 = base64.b64encode(image_data).decode('utf-8')
             
+            # 使用默认提示词如果没有提供
+            query = prompt or "请描述这张图片的内容。"
+            
             # 使用 ollama 库直接调用
             response = ollama.chat(
                 model=self.model_name,
@@ -52,9 +60,127 @@ class OllamaVisionService(BaseService):
                 }]
             )
             
-            return response['message']['content']
+            content = response['message']['content']
+            
+            return {
+                "text": content,
+                "confidence": 1.0,  # Ollama doesn't provide confidence scores
+                "detected_objects": [],  # Basic implementation
+                "metadata": {
+                    "model": self.model_name,
+                    "prompt": query
+                }
+            }
             
         except Exception as e:
             logger.error(f"Error in image analysis: {e}")
             raise
 
+    async def analyze_images(
+        self, 
+        images: List[Union[str, BinaryIO]],
+        prompt: Optional[str] = None,
+        max_tokens: int = 1000
+    ) -> List[Dict[str, Any]]:
+        """Analyze multiple images"""
+        results = []
+        for image in images:
+            result = await self.analyze_image(image, prompt, max_tokens)
+            results.append(result)
+        return results
+
+    async def describe_image(
+        self, 
+        image: Union[str, BinaryIO],
+        detail_level: str = "medium"
+    ) -> Dict[str, Any]:
+        """Generate detailed description of image"""
+        prompts = {
+            "low": "简单描述这张图片。",
+            "medium": "详细描述这张图片的内容、颜色、物体和场景。",
+            "high": "非常详细地描述这张图片，包括所有可见的物体、颜色、纹理、场景、情感和任何其他细节。"
+        }
+        
+        prompt = prompts.get(detail_level, prompts["medium"])
+        result = await self.analyze_image(image, prompt)
+        
+        return {
+            "description": result["text"],
+            "objects": [],  # Basic implementation
+            "scene": "未知",  # Basic implementation
+            "colors": []  # Basic implementation
+        }
+
+    async def extract_text(self, image: Union[str, BinaryIO]) -> Dict[str, Any]:
+        """Extract text from image (OCR)"""
+        result = await self.analyze_image(image, "提取图片中的所有文字内容。")
+        
+        return {
+            "text": result["text"],
+            "confidence": 1.0,
+            "bounding_boxes": [],  # Basic implementation
+            "language": "未知"  # Basic implementation
+        }
+
+    async def detect_objects(
+        self, 
+        image: Union[str, BinaryIO],
+        confidence_threshold: float = 0.5
+    ) -> Dict[str, Any]:
+        """Detect objects in image"""
+        result = await self.analyze_image(image, "识别并列出图片中的所有物体。")
+        
+        return {
+            "objects": [],  # Basic implementation - would need parsing
+            "count": 0,
+            "bounding_boxes": []
+        }
+
+    async def classify_image(
+        self, 
+        image: Union[str, BinaryIO],
+        categories: Optional[List[str]] = None
+    ) -> Dict[str, Any]:
+        """Classify image into categories"""
+        if categories:
+            category_str = "、".join(categories)
+            prompt = f"将这张图片分类到以下类别之一：{category_str}"
+        else:
+            prompt = "这张图片属于什么类别？"
+            
+        result = await self.analyze_image(image, prompt)
+        
+        return {
+            "category": result["text"],
+            "confidence": 1.0,
+            "all_predictions": [{"category": result["text"], "confidence": 1.0}]
+        }
+
+    async def compare_images(
+        self, 
+        image1: Union[str, BinaryIO],
+        image2: Union[str, BinaryIO]
+    ) -> Dict[str, Any]:
+        """Compare two images for similarity"""
+        # For now, analyze each image separately and compare descriptions
+        result1 = await self.analyze_image(image1, "描述这张图片。")
+        result2 = await self.analyze_image(image2, "描述这张图片。")
+        
+        return {
+            "similarity_score": 0.5,  # Basic implementation
+            "differences": "需要进一步分析",
+            "common_elements": "需要进一步分析"
+        }
+
+    def get_supported_formats(self) -> List[str]:
+        """Get list of supported image formats"""
+        return ['jpg', 'jpeg', 'png', 'gif', 'bmp', 'webp']
+
+    def get_max_image_size(self) -> Dict[str, int]:
+        """Get maximum supported image dimensions"""
+        return {"width": 4096, "height": 4096}
+
+    async def close(self):
+        """Cleanup resources"""
+        pass
+