isa-model 0.3.4__py3-none-any.whl → 0.3.6__py3-none-any.whl

isa_model/inference/services/vision/helpers/vision_prompts.py

@@ -0,0 +1,297 @@
+"""
+统一的Vision任务提示词服务
+为不同的Vision模型提供标准化的提示词模板，避免重复代码
+"""
+
+from typing import List, Optional, Dict, Any
+
+
+class VisionPromptService:
+    """Vision任务的统一提示词生成服务"""
+    
+    @staticmethod
+    def get_describe_prompt(detail_level: str = "medium") -> str:
+        """生成图像描述提示词"""
+        detail_prompts = {
+            "brief": "Please provide a brief, one-sentence description of this image.",
+            "medium": "Please provide a detailed description of this image, including main objects, people, setting, and notable details.",
+            "detailed": "Please provide a comprehensive and detailed description of this image, including all visible objects, people, setting, colors, composition, style, mood, and any other notable details or context."
+        }
+        return detail_prompts.get(detail_level, detail_prompts["medium"])
+    
+    @staticmethod
+    def get_extract_text_prompt() -> str:
+        """生成文本提取(OCR)提示词"""
+        return """Please extract ALL text content from this image. Requirements:
+        1. Extract text exactly as it appears
+        2. Preserve formatting, line breaks, and structure
+        3. If there are tables, maintain table structure
+        4. Include headers, captions, and footnotes
+        5. Return as structured JSON with extracted text and layout information
+        
+        Format your response as JSON:
+        {
+            "extracted_text": "full text content",
+            "structured_content": {
+                "headers": [],
+                "paragraphs": [],
+                "tables": [],
+                "other": []
+            }
+        }"""
+    
+    @staticmethod
+    def get_detect_objects_prompt(confidence_threshold: float = 0.5) -> str:
+        """生成物体检测提示词"""
+        return f"""Please identify and locate all objects in this image. For each object:
+        1. Object name/type
+        2. Approximate location (describe position: top-left, center, bottom-right, etc.)
+        3. Size (small, medium, large)
+        4. Confidence level (high, medium, low)
+        
+        Only include objects you're confident about (confidence > {confidence_threshold})
+        
+        Format as JSON:
+        {{
+            "detected_objects": [
+                {{
+                    "name": "object_name",
+                    "location": "position_description", 
+                    "size": "relative_size",
+                    "confidence": "confidence_level"
+                }}
+            ]
+        }}"""
+    
+    @staticmethod
+    def get_detect_ui_elements_prompt(element_types: Optional[List[str]] = None) -> str:
+        """生成UI元素检测提示词"""
+        element_filter = f"Focus on these element types: {', '.join(element_types)}" if element_types else "Identify all UI elements"
+        
+        return f"""Please analyze this user interface image and identify all interactive elements. {element_filter}
+        
+        For each UI element, provide:
+        1. Element type (button, input field, dropdown, link, checkbox, radio button, text area, etc.)
+        2. Text/label content
+        3. Location description
+        4. Interactive state (enabled, disabled, selected, etc.)
+        
+        Format as JSON:
+        {{
+            "ui_elements": [
+                {{
+                    "type": "element_type",
+                    "text": "visible_text",
+                    "location": "position_description",
+                    "state": "element_state",
+                    "confidence": "detection_confidence"
+                }}
+            ]
+        }}"""
+    
+    @staticmethod
+    def get_detect_document_elements_prompt() -> str:
+        """生成文档元素检测提示词"""
+        return """Please analyze this document image and extract its structure and content.
+        
+        Identify and extract:
+        1. Headers and subheaders (with hierarchy level)
+        2. Paragraphs and body text
+        3. Tables (with rows and columns)
+        4. Lists (ordered/unordered)
+        5. Images and captions
+        6. Footnotes and references
+        
+        Format as JSON:
+        {
+            "document_structure": {
+                "title": "document_title",
+                "headers": [
+                    {"level": 1, "text": "header_text", "position": "location"}
+                ],
+                "paragraphs": [
+                    {"text": "paragraph_content", "position": "location"}
+                ],
+                "tables": [
+                    {"rows": [["cell1", "cell2"]], "caption": "table_caption"}
+                ],
+                "lists": [
+                    {"type": "ordered/unordered", "items": ["item1", "item2"]}
+                ]
+            }
+        }"""
+    
+    @staticmethod
+    def get_extract_table_data_prompt(table_format: str = "json", preserve_formatting: bool = True) -> str:
+        """生成表格数据抽取提示词"""
+        format_instructions = {
+            "json": "Return the table data as a JSON structure with arrays for headers and rows",
+            "csv": "Return the table data in CSV format",
+            "markdown": "Return the table data in Markdown table format", 
+            "html": "Return the table data as an HTML table"
+        }
+        
+        format_instruction = format_instructions.get(table_format, format_instructions["json"])
+        formatting_note = "Preserve cell merging, formatting, and styling information" if preserve_formatting else "Extract data in simplified format"
+        
+        return f"""Please extract ALL table data from this image with high precision. {formatting_note}
+        
+        Requirements:
+        1. Identify all tables in the image
+        2. Extract headers, rows, and data accurately
+        3. Maintain data relationships and structure
+        4. Handle merged cells appropriately
+        5. Include any table captions or titles
+        6. {format_instruction}
+        
+        For each table, provide:
+        - Table identifier/caption
+        - Column headers
+        - All row data
+        - Metadata about structure (row/column counts, merged cells)
+        
+        Return as structured JSON:
+        {{
+            "tables": [
+                {{
+                    "table_id": "table_1",
+                    "caption": "table_title_if_any",
+                    "headers": ["Column1", "Column2", "Column3"],
+                    "rows": [
+                        ["data1", "data2", "data3"],
+                        ["data4", "data5", "data6"]
+                    ],
+                    "metadata": {{
+                        "row_count": 2,
+                        "column_count": 3,
+                        "has_headers": true,
+                        "merged_cells": [
+                            {{"row": 0, "col": 0, "rowspan": 1, "colspan": 2}}
+                        ],
+                        "data_types": ["text", "number", "text"]
+                    }}
+                }}
+            ],
+            "extraction_metadata": {{
+                "total_tables": 1,
+                "extraction_confidence": "high",
+                "format": "{table_format}",
+                "preserve_formatting": {str(preserve_formatting).lower()}
+            }}
+        }}
+        
+        Important: 
+        - Be extremely accurate with data extraction
+        - Preserve numbers exactly as they appear
+        - Handle currency, percentages, and special characters correctly
+        - If cells are empty, represent them as empty strings or null
+        - For merged cells, include merge information in metadata"""
+    
+    @staticmethod
+    def get_classify_image_prompt(categories: Optional[List[str]] = None) -> str:
+        """生成图像分类提示词"""
+        if categories:
+            return f"""Please classify this image into one of these categories: {', '.join(categories)}
+            
+            Provide:
+            1. The most appropriate category
+            2. Confidence level (0.0-1.0) 
+            3. Brief reasoning
+            
+            Format as JSON:
+            {{
+                "classification": "selected_category",
+                "confidence": 0.95,
+                "reasoning": "explanation"
+            }}"""
+        else:
+            return """Please classify this image by identifying its main category and subcategory.
+            
+            Provide:
+            1. Main category (e.g., nature, technology, people, etc.)
+            2. Subcategory (more specific classification)
+            3. Confidence level
+            4. Key features that led to this classification
+            
+            Format as JSON:
+            {
+                "main_category": "primary_category",
+                "subcategory": "specific_type", 
+                "confidence": 0.95,
+                "key_features": ["feature1", "feature2"]
+            }"""
+    
+    @staticmethod
+    def get_object_coordinates_prompt(object_name: str) -> str:
+        """生成对象坐标检测提示词"""
+        return f"""Please locate '{object_name}' in this image and provide detailed location information.
+        
+        Provide:
+        1. Whether the object was found
+        2. Detailed position description
+        3. Approximate coordinates (if possible, describe as percentages from top-left)
+        4. Size and boundaries
+        
+        Format as JSON:
+        {{
+            "found": true/false,
+            "object_name": "{object_name}",
+            "location": "detailed_position_description",
+            "coordinates": "approximate_position_as_percentages",
+            "size": "object_size_description",
+            "confidence": "detection_confidence"
+        }}"""
+    
+    @staticmethod
+    def get_compare_images_prompt() -> str:
+        """生成图像比较提示词"""
+        return """Please compare the objects, styles, and content in this image. Highlight similarities and differences.
+        
+        Provide:
+        1. Main similarities
+        2. Key differences
+        3. Style comparison
+        4. Content analysis
+        
+        Format as JSON:
+        {
+            "comparison": {
+                "similarities": ["similarity1", "similarity2"],
+                "differences": ["difference1", "difference2"],
+                "style_analysis": "style_comparison",
+                "content_analysis": "content_comparison"
+            }
+        }"""
+
+
+class VisionPromptMixin:
+    """
+    Mixin类，为Vision服务提供统一的提示词支持
+    任何Vision服务都可以继承这个Mixin来获得标准提示词
+    """
+    
+    def get_task_prompt(self, task: str, **kwargs) -> str:
+        """根据任务类型获取对应的提示词"""
+        if task == "describe":
+            return VisionPromptService.get_describe_prompt(kwargs.get("detail_level", "medium"))
+        elif task == "extract_text":
+            return VisionPromptService.get_extract_text_prompt()
+        elif task == "detect_objects":
+            return VisionPromptService.get_detect_objects_prompt(kwargs.get("confidence_threshold", 0.5))
+        elif task == "detect_ui_elements":
+            return VisionPromptService.get_detect_ui_elements_prompt(kwargs.get("element_types"))
+        elif task == "detect_document_elements":
+            return VisionPromptService.get_detect_document_elements_prompt()
+        elif task == "extract_table_data":
+            return VisionPromptService.get_extract_table_data_prompt(
+                kwargs.get("table_format", "json"), 
+                kwargs.get("preserve_formatting", True)
+            )
+        elif task == "classify":
+            return VisionPromptService.get_classify_image_prompt(kwargs.get("categories"))
+        elif task == "get_coordinates":
+            return VisionPromptService.get_object_coordinates_prompt(kwargs.get("object_name", ""))
+        elif task == "compare":
+            return VisionPromptService.get_compare_images_prompt()
+        else:
+            return "Please analyze this image and provide detailed information."

isa_model/inference/services/vision/openai_vision_service.py

@@ -1,32 +1,31 @@
 from typing import Dict, Any, Union, List, Optional, BinaryIO
-import base64
-import aiohttp
 from openai import AsyncOpenAI
 from tenacity import retry, stop_after_attempt, wait_exponential
 from isa_model.inference.services.vision.base_vision_service import BaseVisionService
-from isa_model.inference.providers.base_provider import BaseProvider
-from isa_model.inference.billing_tracker import ServiceType
+from isa_model.inference.services.vision.helpers.image_utils import prepare_image_base64
+from isa_model.inference.services.vision.helpers.vision_prompts import VisionPromptMixin
+from isa_model.core.types import ServiceType
 import logging
 
 logger = logging.getLogger(__name__)
 
-class OpenAIVisionService(BaseVisionService):
-    """OpenAI Vision service using gpt-4.1-nano with vision capabilities"""
+class OpenAIVisionService(BaseVisionService, VisionPromptMixin):
+    """OpenAI Vision service using centralized config management"""
     
-    def __init__(self, provider: 'BaseProvider', model_name: str = "gpt-4.1-nano"):
-        super().__init__(provider, model_name)
+    def __init__(self, provider_name: str, model_name: str = "gpt-4o-mini", **kwargs):
+        super().__init__(provider_name, model_name, **kwargs)
         
-        # Get full configuration from provider (including sensitive data)
-        provider_config = provider.get_full_config()
+        # Get configuration from centralized config manager
+        provider_config = self.get_provider_config()
         
-        # Initialize AsyncOpenAI client with provider configuration
+        # Initialize AsyncOpenAI client with centralized configuration
         try:
             if not provider_config.get("api_key"):
                 raise ValueError("OpenAI API key not found in provider configuration")
             
             self._client = AsyncOpenAI(
                 api_key=provider_config["api_key"],
-                base_url=provider_config.get("base_url", "https://api.openai.com/v1"),
+                base_url=provider_config.get("api_base_url", "https://api.openai.com/v1"),
                 organization=provider_config.get("organization")
             )
             
@@ -44,31 +43,7 @@ class OpenAIVisionService(BaseVisionService):
         """Get the underlying OpenAI client"""
         return self._client
     
-    async def _download_image(self, image_url: str) -> bytes:
-        """Download image from URL"""
-        async with aiohttp.ClientSession() as session:
-            async with session.get(image_url) as response:
-                if response.status == 200:
-                    return await response.read()
-                else:
-                    raise ValueError(f"Failed to download image from {image_url}: {response.status}")
     
-    def _encode_image(self, image_path_or_data: Union[str, bytes, BinaryIO]) -> str:
-        """Encode image to base64"""
-        if isinstance(image_path_or_data, str):
-            # If it's a file path
-            with open(image_path_or_data, "rb") as image_file:
-                return base64.b64encode(image_file.read()).decode("utf-8")
-        elif hasattr(image_path_or_data, 'read'):
-            # If it's a file-like object (BinaryIO)
-            data = image_path_or_data.read()  # type: ignore
-            if isinstance(data, bytes):
-                return base64.b64encode(data).decode("utf-8")
-            else:
-                raise ValueError("File-like object did not return bytes")
-        else:
-            # If it's bytes data
-            return base64.b64encode(image_path_or_data).decode("utf-8")  # type: ignore
     
     @retry(
         stop=stop_after_attempt(3),
@@ -93,22 +68,8 @@ class OpenAIVisionService(BaseVisionService):
             Dict containing analysis results
         """
         try:
-            # Handle different input types
-            if isinstance(image, str):
-                if image.startswith(('http://', 'https://')):
-                    # Download image from URL
-                    image_bytes = await self._download_image(image)
-                    base64_image = self._encode_image(image_bytes)
-                else:
-                    # File path
-                    base64_image = self._encode_image(image)
-            else:
-                # BinaryIO or bytes data
-                if hasattr(image, 'read'):
-                    image_data = image.read()
-                else:
-                    image_data = image
-                base64_image = self._encode_image(image_data)
+            # Use unified image processing from image_utils
+            base64_image = prepare_image_base64(image)
             
             # Use default prompt if none provided
             if prompt is None:
@@ -140,7 +101,7 @@ class OpenAIVisionService(BaseVisionService):
             
             # Track usage for billing
             if response.usage:
-                self._track_usage(
+                await self._track_usage(
                     service_type=ServiceType.VISION,
                     operation="image_analysis",
                     input_tokens=response.usage.prompt_tokens,
@@ -150,14 +111,36 @@ class OpenAIVisionService(BaseVisionService):
             
             content = response.choices[0].message.content or ""
             
+            # 尝试解析JSON响应（对于结构化任务）
+            try:
+                import json
+                # 检查响应是否是JSON格式
+                if content.strip().startswith('{') and content.strip().endswith('}'):
+                    parsed_json = json.loads(content)
+                    return {
+                        "text": content,
+                        "parsed_data": parsed_json,
+                        "confidence": 1.0,
+                        "metadata": {
+                            "model": self.model_name,
+                            "prompt": prompt[:100],
+                            "tokens_used": response.usage.total_tokens if response.usage else 0,
+                            "response_format": "json"
+                        }
+                    }
+            except json.JSONDecodeError:
+                pass
+            
+            # 标准文本响应
             return {
                 "text": content,
                 "confidence": 1.0,  # OpenAI doesn't provide confidence scores
-                "detected_objects": [],  # Would need separate object detection
+                "detected_objects": [],  # Populated by specific detection methods
                 "metadata": {
                     "model": self.model_name,
-                    "prompt": prompt,
-                    "tokens_used": response.usage.total_tokens if response.usage else 0
+                    "prompt": prompt[:100],
+                    "tokens_used": response.usage.total_tokens if response.usage else 0,
+                    "response_format": "text"
                 }
             }
             
@@ -165,148 +148,104 @@ class OpenAIVisionService(BaseVisionService):
             logger.error(f"Error in image analysis: {e}")
             raise
     
-    async def analyze_images(
-        self, 
-        images: List[Union[str, BinaryIO]],
-        prompt: Optional[str] = None,
-        max_tokens: int = 1000
-    ) -> List[Dict[str, Any]]:
-        """Analyze multiple images"""
-        results = []
-        for image in images:
-            result = await self.analyze_image(image, prompt, max_tokens)
-            results.append(result)
-        return results
+    # ==================== 基于提示词的智能功能实现 ====================
+    # OpenAI通过改变提示词就能实现大部分Vision功能
+    # 使用统一的VisionPromptMixin提供标准提示词
     
+    # 重写其他方法以使用智能提示词
     async def describe_image(
         self, 
         image: Union[str, BinaryIO],
         detail_level: str = "medium"
     ) -> Dict[str, Any]:
-        """Generate detailed description of image"""
-        detail_prompts = {
-            "low": "Briefly describe what you see in this image.",
-            "medium": "Describe what you see in this image in detail, including objects, colors, and scene.",
-            "high": "Provide a comprehensive and detailed description of this image, including all visible objects, their positions, colors, textures, lighting, composition, and any text or symbols present."
-        }
-        
-        prompt = detail_prompts.get(detail_level, detail_prompts["medium"])
-        result = await self.analyze_image(image, prompt, 1500)
-        
-        return {
-            "description": result["text"],
-            "objects": [],  # Would need object detection API
-            "scene": result["text"],  # Use same description
-            "colors": [],  # Would need color analysis
-            "detail_level": detail_level,
-            "metadata": result["metadata"]
-        }
+        """
+        图像描述 - 使用专门提示词
+        """
+        prompt = self.get_task_prompt("describe", detail_level=detail_level)
+        return await self.analyze_image(image, prompt)
     
     async def extract_text(self, image: Union[str, BinaryIO]) -> Dict[str, Any]:
-        """Extract text from image (OCR)"""
-        prompt = "Extract all text visible in this image. Provide only the text content, maintaining the original structure and formatting as much as possible."
-        result = await self.analyze_image(image, prompt, 1000)
+        """
+        文本提取(OCR) - 使用专门提示词
+        """
+        prompt = self.get_task_prompt("extract_text")
         
-        return {
-            "text": result["text"],
-            "confidence": 1.0,
-            "bounding_boxes": [],  # OpenAI vision doesn't provide bounding boxes
-            "language": "unknown",  # Would need language detection
-            "metadata": result["metadata"]
-        }
+        return await self.analyze_image(image, prompt)
     
     async def detect_objects(
         self, 
         image: Union[str, BinaryIO],
         confidence_threshold: float = 0.5
     ) -> Dict[str, Any]:
-        """Detect objects in image"""
-        prompt = "List all objects visible in this image. For each object, provide the object name and a brief description of its location in the image."
-        result = await self.analyze_image(image, prompt, 1000)
+        """
+        物体检测 - 使用专门提示词
+        """
+        prompt = self.get_task_prompt("detect_objects", confidence_threshold=confidence_threshold)
+        
+        return await self.analyze_image(image, prompt)
+    
+    async def detect_ui_elements(
+        self,
+        image: Union[str, BinaryIO],
+        element_types: Optional[List[str]] = None,
+        confidence_threshold: float = 0.5
+    ) -> Dict[str, Any]:
+        """
+        UI元素检测 - 使用专门提示词
+        """
+        prompt = self.get_task_prompt("detect_ui_elements", element_types=element_types, confidence_threshold=confidence_threshold)
         
-        # Parse the response to extract object information
-        objects = []
-        lines = result["text"].split('\n')
-        for line in lines:
-            line = line.strip()
-            if line and not line.startswith(('In this image', 'The image shows', 'I can see')):
-                objects.append({
-                    "label": line,
-                    "confidence": 1.0  # OpenAI doesn't provide confidence scores
-                })
+        return await self.analyze_image(image, prompt)
+    
+    async def detect_document_elements(
+        self,
+        image: Union[str, BinaryIO],
+        element_types: Optional[List[str]] = None,
+        confidence_threshold: float = 0.5
+    ) -> Dict[str, Any]:
+        """
+        文档元素检测 - 使用专门提示词  
+        """
+        prompt = self.get_task_prompt("detect_document_elements", element_types=element_types, confidence_threshold=confidence_threshold)
         
-        return {
-            "objects": objects,
-            "count": len(objects),
-            "bounding_boxes": [],  # Not available with current API
-            "metadata": result["metadata"]
-        }
+        return await self.analyze_image(image, prompt)
     
     async def classify_image(
         self, 
         image: Union[str, BinaryIO],
         categories: Optional[List[str]] = None
     ) -> Dict[str, Any]:
-        """Classify image into categories"""
-        if categories:
-            category_list = ", ".join(categories)
-            prompt = f"Classify this image into one of these categories: {category_list}. Respond with only the most appropriate category name."
-        else:
-            prompt = "What category best describes this image? Provide a single category name."
-        
-        result = await self.analyze_image(image, prompt, 100)
-        category = result["text"].strip()
+        """
+        图像分类 - 使用专门提示词
+        """
+        prompt = self.get_task_prompt("classify", categories=categories)
         
-        return {
-            "category": category,
-            "confidence": 1.0,
-            "all_predictions": [{"category": category, "confidence": 1.0}],
-            "metadata": result["metadata"]
-        }
+        return await self.analyze_image(image, prompt)
     
-    async def compare_images(
-        self, 
-        image1: Union[str, BinaryIO],
-        image2: Union[str, BinaryIO]
+    async def get_object_coordinates(
+        self,
+        image: Union[str, BinaryIO],
+        object_name: str
     ) -> Dict[str, Any]:
-        """Compare two images for similarity"""
-        # For now, analyze both images separately and compare descriptions
-        result1 = await self.analyze_image(image1, "Describe this image in detail.")
-        result2 = await self.analyze_image(image2, "Describe this image in detail.")
-        
-        # Use LLM to compare the descriptions
-        comparison_prompt = f"Compare these two image descriptions and provide a similarity analysis:\n\nImage 1: {result1['text']}\n\nImage 2: {result2['text']}\n\nProvide: 1) A similarity score from 0.0 to 1.0, 2) Key differences, 3) Common elements."
-        
-        comparison_result = await self._client.chat.completions.create(
-            model=self.model_name,
-            messages=[{"role": "user", "content": comparison_prompt}],
-            max_tokens=500,
-            temperature=0.3
-        )
-        
-        comparison_text = comparison_result.choices[0].message.content or ""
+        """
+        获取对象坐标 - 使用专门提示词
+        """
+        prompt = self.get_task_prompt("get_coordinates", object_name=object_name)
         
-        return {
-            "similarity_score": 0.5,  # Would need better parsing to extract actual score
-            "differences": comparison_text,
-            "common_elements": comparison_text,
-            "metadata": {
-                "model": self.model_name,
-                "comparison_method": "description_based"
-            }
-        }
-    
-    def get_supported_formats(self) -> List[str]:
-        """Get list of supported image formats"""
-        return ['jpg', 'jpeg', 'png', 'gif', 'webp']
+        return await self.analyze_image(image, prompt)
     
-    def get_max_image_size(self) -> Dict[str, int]:
-        """Get maximum supported image dimensions"""
-        return {
-            "width": 2048,
-            "height": 2048,
-            "file_size_mb": 20
-        }
+    async def extract_table_data(
+        self,
+        image: Union[str, BinaryIO],
+        table_format: str = "json",
+        preserve_formatting: bool = True
+    ) -> Dict[str, Any]:
+        """
+        表格数据结构化抽取 - 使用专门的表格抽取提示词
+        """
+        prompt = self.get_task_prompt("extract_table_data", table_format=table_format, preserve_formatting=preserve_formatting)
+        
+        return await self.analyze_image(image, prompt)
     
     async def close(self):
         """Clean up resources"""