isa-model 0.3.3__py3-none-any.whl → 0.3.5__py3-none-any.whl

isa_model/inference/services/vision/base_image_gen_service.py

@@ -68,40 +68,6 @@ class BaseImageGenService(BaseService):
         """
         pass
     
-    @abstractmethod
-    async def generate_image_to_file(
-        self, 
-        prompt: str,
-        output_path: str,
-        negative_prompt: Optional[str] = None,
-        width: int = 512,
-        height: int = 512,
-        num_inference_steps: int = 20,
-        guidance_scale: float = 7.5,
-        seed: Optional[int] = None
-    ) -> Dict[str, Any]:
-        """
-        Generate image and save directly to file
-        
-        Args:
-            prompt: Text description of the desired image
-            output_path: Path to save the generated image
-            negative_prompt: Text describing what to avoid in the image
-            width: Image width in pixels
-            height: Image height in pixels
-            num_inference_steps: Number of denoising steps
-            guidance_scale: How closely to follow the prompt
-            seed: Random seed for reproducible results
-            
-        Returns:
-            Dict containing generation results with keys:
-            - file_path: Path to saved image
-            - width: Image width
-            - height: Image height
-            - seed: Seed used for generation
-        """
-        pass
-    
     @abstractmethod
     async def image_to_image(
         self,

isa_model/inference/services/vision/base_vision_service.py

@@ -5,6 +5,28 @@ from isa_model.inference.services.base_service import BaseService
 class BaseVisionService(BaseService):
     """Base class for vision understanding services"""
     
+    @abstractmethod
+    async def invoke(
+        self, 
+        image: Union[str, BinaryIO],
+        prompt: Optional[str] = None,
+        task: Optional[str] = None,
+        **kwargs
+    ) -> Dict[str, Any]:
+        """
+        Unified invoke method for all vision operations
+        
+        Args:
+            image: Path to image file or image data
+            prompt: Optional text prompt/question about the image
+            task: Task type (analyze, describe, extract_text, detect_objects, etc.)
+            **kwargs: Additional task-specific parameters
+            
+        Returns:
+            Dict containing task results
+        """
+        pass
+    
     @abstractmethod
     async def analyze_image(
         self, 
@@ -103,9 +125,31 @@ class BaseVisionService(BaseService):
             
         Returns:
             Dict containing detection results with keys:
-            - objects: List of detected objects with labels and confidence
+            - objects: List of detected objects with labels, confidence, and coordinates
             - count: Number of objects detected
-            - bounding_boxes: Object locations (if available)
+            - bounding_boxes: Object locations with coordinates
+        """
+        pass
+    
+    @abstractmethod
+    async def get_object_coordinates(
+        self,
+        image: Union[str, BinaryIO],
+        object_name: str
+    ) -> Dict[str, Any]:
+        """
+        Get coordinates of a specific object in the image
+        
+        Args:
+            image: Path to image file or image data
+            object_name: Name of the object to locate
+            
+        Returns:
+            Dict containing coordinate results with keys:
+            - found: Boolean indicating if object was found
+            - center_coordinates: List [x, y] with pixel coordinates of center point
+            - confidence: Confidence score for the detection
+            - description: Description of the object location
         """
         pass
     

isa_model/inference/services/vision/isA_vision_service.py

@@ -0,0 +1,402 @@
+"""
+ISA Vision Service
+
+Connects to self-hosted Modal UI detection service
+Provides vision capabilities using our deployed models
+"""
+
+import modal
+import base64
+import io
+import logging
+from typing import Dict, Any, List, Union, Optional, BinaryIO
+from PIL import Image
+
+from .base_vision_service import BaseVisionService
+
+logger = logging.getLogger(__name__)
+
+class ISAVisionService(BaseVisionService):
+    """ISA Vision Service using Modal backend"""
+    
+    def __init__(self, provider, model_name: str):
+        super().__init__(provider, model_name)
+        self.ui_app = None
+        self.doc_app = None
+        self._initialize_modal_connections()
+    
+    def _initialize_modal_connections(self):
+        """Initialize connections to Modal services"""
+        try:
+            # Connect to UI detection service
+            self.ui_app = modal.App.lookup("isa-vision-ui", create_if_missing=False)
+            logger.info(" Connected to UI detection service")
+        except Exception as e:
+            logger.warning(f"� UI service not available: {e}")
+            self.ui_app = None
+        
+        try:
+            # Connect to document analysis service (when deployed)
+            self.doc_app = modal.App.lookup("isa-vision-doc", create_if_missing=False)
+            logger.info(" Connected to document analysis service")
+        except Exception as e:
+            logger.warning(f"� Document service not available: {e}")
+            self.doc_app = None
+    
+    async def invoke(
+        self, 
+        image: Union[str, BinaryIO],
+        prompt: Optional[str] = None,
+        task: Optional[str] = None,
+        **kwargs
+    ) -> Dict[str, Any]:
+        """
+        Unified invoke method for all vision operations
+        """
+        if task == "detect_ui" or task == "ui_analysis":
+            return await self.detect_objects(image, **kwargs)
+        elif task == "extract_text" or task == "ocr":
+            return await self.extract_text(image)
+        elif task == "analyze_document":
+            return await self._analyze_document(image)
+        else:
+            return await self.analyze_image(image, prompt, **kwargs)
+    
+    async def analyze_image(
+        self, 
+        image: Union[str, BinaryIO],
+        prompt: Optional[str] = None,
+        max_tokens: int = 1000
+    ) -> Dict[str, Any]:
+        """Analyze image using UI detection service"""
+        
+        if not self.ui_app:
+            return {
+                'error': 'UI detection service not available',
+                'success': False
+            }
+        
+        try:
+            # Convert image to base64
+            image_b64 = self._encode_image(image)
+            
+            # Call Modal UI detection service using from_name (new API)
+            ui_detector = modal.Cls.from_name("isa-vision-ui", "UIDetectionService")
+            result = ui_detector().detect_ui_elements.remote(image_b64)
+            
+            if result.get('success'):
+                return {
+                    'success': True,
+                    'service': 'isa-vision',
+                    'text': f"Detected {result.get('element_count', 0)} UI elements",
+                    'detected_objects': result.get('ui_elements', []),
+                    'confidence': 0.9,
+                    'metadata': {
+                        'processing_time': result.get('processing_time'),
+                        'detection_method': result.get('detection_method'),
+                        'model_info': result.get('model_info')
+                    }
+                }
+            else:
+                return {
+                    'success': False,
+                    'error': result.get('error', 'Unknown error'),
+                    'service': 'isa-vision'
+                }
+                
+        except Exception as e:
+            logger.error(f"Image analysis failed: {e}")
+            return {
+                'success': False,
+                'error': str(e),
+                'service': 'isa-vision'
+            }
+    
+    async def analyze_images(
+        self, 
+        images: List[Union[str, BinaryIO]],
+        prompt: Optional[str] = None,
+        max_tokens: int = 1000
+    ) -> List[Dict[str, Any]]:
+        """Analyze multiple images"""
+        results = []
+        for image in images:
+            result = await self.analyze_image(image, prompt, max_tokens)
+            results.append(result)
+        return results
+    
+    async def describe_image(
+        self, 
+        image: Union[str, BinaryIO],
+        detail_level: str = "medium"
+    ) -> Dict[str, Any]:
+        """Generate description using UI detection"""
+        result = await self.analyze_image(image)
+        
+        if result.get('success'):
+            objects = result.get('detected_objects', [])
+            description = f"This appears to be a user interface with {len(objects)} interactive elements. "
+            
+            if objects:
+                element_types = list(set([obj.get('type', 'element') for obj in objects]))
+                description += f"The interface contains: {', '.join(element_types)}."
+            
+            return {
+                'success': True,
+                'description': description,
+                'objects': objects,
+                'scene': 'User Interface',
+                'colors': ['unknown']  # Could be enhanced with color detection
+            }
+        else:
+            return result
+    
+    async def extract_text(self, image: Union[str, BinaryIO]) -> Dict[str, Any]:
+        """Extract text using document analysis service"""
+        
+        if not self.doc_app:
+            # Fallback to UI service for basic text detection
+            return await self._extract_text_fallback(image)
+        
+        try:
+            # Convert image to base64
+            image_b64 = self._encode_image(image)
+            
+            # Call Modal document analysis service using from_name (new API)
+            doc_analyzer = modal.Cls.from_name("isa-vision-doc", "DocumentAnalysisService")
+            result = doc_analyzer().extract_text.remote(image_b64)
+            
+            if result.get('success'):
+                text_results = result.get('text_results', [])
+                all_text = ' '.join([item.get('text', '') for item in text_results])
+                
+                return {
+                    'success': True,
+                    'service': 'isa-vision-doc',
+                    'text': all_text,
+                    'confidence': sum([item.get('confidence', 0) for item in text_results]) / len(text_results) if text_results else 0,
+                    'bounding_boxes': [item.get('bbox') for item in text_results],
+                    'language': 'auto-detected',
+                    'metadata': {
+                        'processing_time': result.get('processing_time'),
+                        'text_count': result.get('text_count')
+                    }
+                }
+            else:
+                return {
+                    'success': False,
+                    'error': result.get('error', 'OCR failed'),
+                    'service': 'isa-vision-doc'
+                }
+                
+        except Exception as e:
+            logger.error(f"Text extraction failed: {e}")
+            return {
+                'success': False,
+                'error': str(e),
+                'service': 'isa-vision-doc'
+            }
+    
+    async def detect_objects(
+        self, 
+        image: Union[str, BinaryIO],
+        confidence_threshold: float = 0.5
+    ) -> Dict[str, Any]:
+        """Detect UI elements using UI detection service"""
+        
+        result = await self.analyze_image(image)
+        
+        if result.get('success'):
+            objects = result.get('detected_objects', [])
+            # Filter by confidence threshold
+            filtered_objects = [obj for obj in objects if obj.get('confidence', 0) >= confidence_threshold]
+            
+            return {
+                'success': True,
+                'service': 'isa-vision-ui',
+                'objects': filtered_objects,
+                'count': len(filtered_objects),
+                'bounding_boxes': [obj.get('bbox') for obj in filtered_objects],
+                'metadata': result.get('metadata', {})
+            }
+        else:
+            return result
+    
+    async def get_object_coordinates(
+        self,
+        image: Union[str, BinaryIO],
+        object_name: str
+    ) -> Dict[str, Any]:
+        """Get coordinates of specific UI element"""
+        
+        detection_result = await self.detect_objects(image)
+        
+        if not detection_result.get('success'):
+            return detection_result
+        
+        objects = detection_result.get('objects', [])
+        
+        # Look for object by name/type
+        for obj in objects:
+            obj_type = obj.get('type', '').lower()
+            obj_content = obj.get('content', '').lower()
+            
+            if object_name.lower() in obj_type or object_name.lower() in obj_content:
+                return {
+                    'success': True,
+                    'found': True,
+                    'center_coordinates': obj.get('center', [0, 0]),
+                    'confidence': obj.get('confidence', 0),
+                    'description': f"Found {obj.get('type')} at center coordinates",
+                    'object_info': obj
+                }
+        
+        return {
+            'success': True,
+            'found': False,
+            'center_coordinates': [0, 0],
+            'confidence': 0,
+            'description': f"Object '{object_name}' not found in image"
+        }
+    
+    async def classify_image(
+        self, 
+        image: Union[str, BinaryIO],
+        categories: Optional[List[str]] = None
+    ) -> Dict[str, Any]:
+        """Classify image type"""
+        
+        result = await self.analyze_image(image)
+        
+        if result.get('success'):
+            objects = result.get('detected_objects', [])
+            
+            # Simple classification based on detected UI elements
+            if objects:
+                category = "user_interface"
+                confidence = 0.9
+            else:
+                category = "unknown"
+                confidence = 0.1
+            
+            return {
+                'success': True,
+                'category': category,
+                'confidence': confidence,
+                'all_predictions': [
+                    {'category': category, 'confidence': confidence}
+                ]
+            }
+        else:
+            return result
+    
+    async def compare_images(
+        self, 
+        image1: Union[str, BinaryIO],
+        image2: Union[str, BinaryIO]
+    ) -> Dict[str, Any]:
+        """Compare two images based on UI elements"""
+        
+        result1 = await self.analyze_image(image1)
+        result2 = await self.analyze_image(image2)
+        
+        if not (result1.get('success') and result2.get('success')):
+            return {
+                'success': False,
+                'error': 'Failed to analyze one or both images'
+            }
+        
+        objects1 = result1.get('detected_objects', [])
+        objects2 = result2.get('detected_objects', [])
+        
+        # Simple comparison based on element counts and types
+        count_diff = abs(len(objects1) - len(objects2))
+        types1 = set([obj.get('type') for obj in objects1])
+        types2 = set([obj.get('type') for obj in objects2])
+        
+        common_types = types1.intersection(types2)
+        unique_types = types1.symmetric_difference(types2)
+        
+        similarity_score = len(common_types) / max(len(types1.union(types2)), 1)
+        
+        return {
+            'success': True,
+            'similarity_score': similarity_score,
+            'differences': f"Different element types: {list(unique_types)}",
+            'common_elements': f"Common element types: {list(common_types)}",
+            'metadata': {
+                'elements_count_1': len(objects1),
+                'elements_count_2': len(objects2),
+                'count_difference': count_diff
+            }
+        }
+    
+    def get_supported_formats(self) -> List[str]:
+        """Get supported image formats"""
+        return ['jpg', 'jpeg', 'png', 'bmp', 'gif', 'tiff']
+    
+    def get_max_image_size(self) -> Dict[str, int]:
+        """Get maximum image dimensions"""
+        return {'width': 4096, 'height': 4096}
+    
+    async def close(self):
+        """Cleanup resources"""
+        # Modal connections don't need explicit cleanup
+        pass
+    
+    # Helper methods
+    
+    def _encode_image(self, image: Union[str, BinaryIO]) -> str:
+        """Convert image to base64 string"""
+        if isinstance(image, str):
+            # File path
+            with open(image, 'rb') as f:
+                image_data = f.read()
+        else:
+            # Binary data
+            if hasattr(image, 'read'):
+                image_data = image.read()
+            else:
+                # Assume it's bytes
+                image_data = bytes(image) if not isinstance(image, bytes) else image
+        
+        return base64.b64encode(image_data).decode('utf-8')
+    
+    async def _extract_text_fallback(self, image: Union[str, BinaryIO]) -> Dict[str, Any]:
+        """Fallback OCR using UI service (basic text detection)"""
+        # For now, return placeholder
+        return {
+            'success': False,
+            'error': 'OCR service not available, deploy document analysis service',
+            'text': '',
+            'confidence': 0,
+            'service': 'isa-vision-fallback'
+        }
+    
+    async def _analyze_document(self, image: Union[str, BinaryIO]) -> Dict[str, Any]:
+        """Analyze document with tables and OCR"""
+        
+        if not self.doc_app:
+            return {
+                'success': False,
+                'error': 'Document analysis service not deployed',
+                'service': 'isa-vision-doc'
+            }
+        
+        try:
+            # Convert image to base64
+            image_b64 = self._encode_image(image)
+            
+            # Call Modal document analysis service using from_name (new API)
+            doc_analyzer = modal.Cls.from_name("isa-vision-doc", "DocumentAnalysisService")
+            result = doc_analyzer().analyze_document_complete.remote(image_b64)
+            
+            return result
+            
+        except Exception as e:
+            logger.error(f"Document analysis failed: {e}")
+            return {
+                'success': False,
+                'error': str(e),
+                'service': 'isa-vision-doc'
+            }

isa_model/inference/services/vision/openai_vision_service.py

@@ -70,6 +70,32 @@ class OpenAIVisionService(BaseVisionService):
             # If it's bytes data
             return base64.b64encode(image_path_or_data).decode("utf-8")  # type: ignore
     
+    async def invoke(
+        self, 
+        image: Union[str, BinaryIO],
+        prompt: Optional[str] = None,
+        task: Optional[str] = None,
+        **kwargs
+    ) -> Dict[str, Any]:
+        """
+        Unified invoke method for all vision operations
+        """
+        task = task or "analyze"
+        
+        if task == "analyze":
+            return await self.analyze_image(image, prompt, kwargs.get("max_tokens", 1000))
+        elif task == "describe":
+            return await self.describe_image(image, kwargs.get("detail_level", "medium"))
+        elif task == "extract_text":
+            return await self.extract_text(image)
+        elif task == "detect_objects":
+            return await self.detect_objects(image, kwargs.get("confidence_threshold", 0.5))
+        elif task == "classify":
+            return await self.classify_image(image, kwargs.get("categories"))
+        else:
+            # Default to analyze_image for unknown tasks
+            return await self.analyze_image(image, prompt, kwargs.get("max_tokens", 1000))
+    
     @retry(
         stop=stop_after_attempt(3),
         wait=wait_exponential(multiplier=1, min=4, max=10),
@@ -221,24 +247,140 @@ class OpenAIVisionService(BaseVisionService):
         confidence_threshold: float = 0.5
     ) -> Dict[str, Any]:
         """Detect objects in image"""
-        prompt = "List all objects visible in this image. For each object, provide the object name and a brief description of its location in the image."
-        result = await self.analyze_image(image, prompt, 1000)
+        prompt = """List all objects visible in this image. For each object, provide:
+1. Object name
+2. Approximate location as percentages from top-left corner (x%, y%)
+3. Approximate size as percentages of image dimensions (width%, height%)
+4. Brief description
+
+Format each object as: "ObjectName: x=X%, y=Y%, width=W%, height=H% - Description"
+
+Example: "Car: x=25%, y=40%, width=15%, height=12% - Red sedan in the center"
+"""
+        result = await self.analyze_image(image, prompt, 1500)
         
-        # Parse the response to extract object information
+        # Parse the response to extract object information with coordinates
         objects = []
+        bounding_boxes = []
         lines = result["text"].split('\n')
+        
         for line in lines:
             line = line.strip()
-            if line and not line.startswith(('In this image', 'The image shows', 'I can see')):
-                objects.append({
-                    "label": line,
-                    "confidence": 1.0  # OpenAI doesn't provide confidence scores
-                })
+            if line and ':' in line and ('x=' in line or 'width=' in line):
+                try:
+                    # Extract object name and details
+                    parts = line.split(':', 1)
+                    if len(parts) == 2:
+                        object_name = parts[0].strip()
+                        details = parts[1].strip()
+                        
+                        # Extract coordinates using regex-like parsing
+                        coords = {}
+                        for param in ['x', 'y', 'width', 'height']:
+                            param_pattern = f"{param}="
+                            if param_pattern in details:
+                                start_idx = details.find(param_pattern) + len(param_pattern)
+                                end_idx = details.find('%', start_idx)
+                                if end_idx > start_idx:
+                                    try:
+                                        value = float(details[start_idx:end_idx])
+                                        coords[param] = value
+                                    except ValueError:
+                                        continue
+                        
+                        # Extract description (after the coordinates)
+                        desc_start = details.find(' - ')
+                        description = details[desc_start + 3:] if desc_start != -1 else details
+                        
+                        objects.append({
+                            "label": object_name,
+                            "confidence": 1.0,
+                            "coordinates": coords,
+                            "description": description
+                        })
+                        
+                        # Add bounding box if we have coordinates
+                        if all(k in coords for k in ['x', 'y', 'width', 'height']):
+                            bounding_boxes.append({
+                                "label": object_name,
+                                "x_percent": coords['x'],
+                                "y_percent": coords['y'],
+                                "width_percent": coords['width'],
+                                "height_percent": coords['height']
+                            })
+                            
+                except Exception:
+                    # Fallback for objects that don't match expected format
+                    objects.append({
+                        "label": line,
+                        "confidence": 1.0,
+                        "coordinates": {},
+                        "description": line
+                    })
         
         return {
             "objects": objects,
             "count": len(objects),
-            "bounding_boxes": [],  # Not available with current API
+            "bounding_boxes": bounding_boxes,
+            "metadata": result["metadata"]
+        }
+    
+    async def get_object_coordinates(
+        self,
+        image: Union[str, BinaryIO],
+        object_name: str
+    ) -> Dict[str, Any]:
+        """Get coordinates of a specific object in the image"""
+        prompt = f"""Locate the {object_name} in this image and return its center coordinates as [x, y] pixels.
+
+Respond in this exact format:
+FOUND: YES/NO
+CENTER: [x, y]
+DESCRIPTION: [Brief description]
+
+If found, provide the pixel coordinates of the center point.
+If not found, explain why.
+
+Example:
+FOUND: YES
+CENTER: [640, 360]
+DESCRIPTION: Blue login button in the center-left area
+"""
+        
+        result = await self.analyze_image(image, prompt, 300)
+        response_text = result["text"]
+        
+        # Parse the structured response
+        found = False
+        center_coords = None
+        description = ""
+        
+        lines = response_text.split('\n')
+        for line in lines:
+            line = line.strip()
+            if line.startswith('FOUND:'):
+                found = 'YES' in line.upper()
+            elif line.startswith('CENTER:') and found:
+                # Extract center coordinates [x, y]
+                coords_text = line.replace('CENTER:', '').strip()
+                try:
+                    # Remove brackets and split
+                    coords_text = coords_text.replace('[', '').replace(']', '')
+                    if ',' in coords_text:
+                        x_str, y_str = coords_text.split(',')
+                        x = int(float(x_str.strip()))
+                        y = int(float(y_str.strip()))
+                        center_coords = [x, y]
+                except (ValueError, IndexError):
+                    pass
+            elif line.startswith('DESCRIPTION:'):
+                description = line.replace('DESCRIPTION:', '').strip()
+        
+        return {
+            "found": found,
+            "center_coordinates": center_coords,
+            "confidence": 1.0 if found else 0.0,
+            "description": description,
             "metadata": result["metadata"]
         }