isa-model 0.3.91__py3-none-any.whl → 0.4.3__py3-none-any.whl

isa_model/deployment/cloud/modal/isa_vision_ui_service.py

@@ -1,406 +0,0 @@
-"""
-ISA Vision UI Service
-
-Specialized service for UI element detection using OmniParser v2.0
-Fallback to YOLOv8 for general object detection
-"""
-
-import modal
-import torch
-import base64
-import io
-import numpy as np
-from PIL import Image
-from typing import Dict, List, Optional, Any
-import time
-import json
-import os
-import logging
-
-# Define Modal application
-app = modal.App("isa-vision-ui")
-
-# Download UI detection models
-def download_ui_models():
-    """Download UI detection models"""
-    from huggingface_hub import snapshot_download
-    
-    print("📦 Downloading UI detection models...")
-    os.makedirs("/models", exist_ok=True)
-    
-    # Download OmniParser v2.0
-    try:
-        snapshot_download(
-            repo_id="microsoft/OmniParser-v2.0",
-            local_dir="/models/omniparser-v2",
-            allow_patterns=["**/*.pt", "**/*.pth", "**/*.bin", "**/*.json", "**/*.safetensors"]
-        )
-        print("✅ OmniParser v2.0 downloaded")
-    except Exception as e:
-        print(f"⚠️ OmniParser v2.0 download failed: {e}")
-    
-    # Download YOLOv8 (fallback)
-    try:
-        from ultralytics import YOLO
-        model = YOLO('yolov8n.pt')
-        print("✅ YOLOv8 fallback model downloaded")
-    except Exception as e:
-        print(f"⚠️ YOLOv8 download failed: {e}")
-    
-    print("📦 UI models download completed")
-
-# Define Modal container image
-image = (
-    modal.Image.debian_slim(python_version="3.11")
-    .pip_install([
-        # Core AI libraries
-        "torch>=2.0.0",
-        "torchvision", 
-        "transformers>=4.35.0",
-        "ultralytics>=8.0.43",
-        "huggingface_hub",
-        "accelerate",
-        
-        # Image processing
-        "pillow>=10.0.1",
-        "opencv-python-headless",
-        "numpy>=1.24.3",
-        
-        # HTTP libraries
-        "httpx>=0.26.0",
-        "requests",
-        
-        # Utilities
-        "pydantic>=2.0.0",
-        "python-dotenv",
-    ])
-    .run_function(download_ui_models)
-    .env({"TRANSFORMERS_CACHE": "/models"})
-)
-
-# UI Detection Service
-@app.cls(
-    gpu="T4",
-    image=image,
-    memory=16384,  # 16GB RAM
-    timeout=1800,  # 30 minutes
-    scaledown_window=60,   # 1 minute idle timeout
-    min_containers=0,  # Scale to zero to save costs
-)
-class UIDetectionService:
-    """
-    UI Element Detection Service
-    
-    Provides fast UI element detection using OmniParser v2.0
-    Falls back to YOLOv8 for general object detection
-    """
-    
-    def __init__(self):
-        self.models = {}
-        self.logger = logging.getLogger(__name__)
-        
-    @modal.enter()
-    def load_models(self):
-        """Load UI detection models on container startup"""
-        print("🚀 Loading UI detection models...")
-        start_time = time.time()
-        
-        # Try to load OmniParser first
-        try:
-            self._load_omniparser()
-        except Exception as e:
-            print(f"⚠️ OmniParser failed to load: {e}")
-            # Fall back to YOLOv8
-            self._load_yolo_fallback()
-        
-        load_time = time.time() - start_time
-        print(f"✅ UI detection models loaded in {load_time:.2f}s")
-        
-    def _load_omniparser(self):
-        """Load OmniParser model"""
-        # Placeholder for actual OmniParser loading
-        # In practice, you would load the actual OmniParser model here
-        print("📱 Loading OmniParser v2.0...")
-        self.models['ui_detector'] = "omniparser_placeholder"
-        print("✅ OmniParser v2.0 loaded")
-        
-    def _load_yolo_fallback(self):
-        """Load YOLOv8 as fallback"""
-        from ultralytics import YOLO
-        
-        print("🔄 Loading YOLOv8 fallback...")
-        yolo_model = YOLO('yolov8n.pt')
-        self.models['detector'] = yolo_model
-        print("✅ YOLOv8 fallback loaded")
-    
-    @modal.method()
-    def detect_ui_elements(self, image_b64: str, detection_type: str = "ui") -> Dict[str, Any]:
-        """
-        Detect UI elements in image
-        
-        Args:
-            image_b64: Base64 encoded image
-            detection_type: Type of detection ("ui" or "general")
-            
-        Returns:
-            Detection results with UI elements
-        """
-        start_time = time.time()
-        
-        try:
-            # Decode image
-            image = self._decode_image(image_b64)
-            image_np = np.array(image)
-            
-            # Perform detection based on available models
-            if 'ui_detector' in self.models:
-                ui_elements = self._omniparser_detection(image_np)
-                detection_method = "omniparser"
-            elif 'detector' in self.models:
-                ui_elements = self._yolo_detection(image_np)
-                detection_method = "yolo_fallback"
-            else:
-                ui_elements = self._opencv_fallback(image_np)
-                detection_method = "opencv_fallback"
-            
-            processing_time = time.time() - start_time
-            
-            return {
-                'success': True,
-                'service': 'isa-vision-ui',
-                'ui_elements': ui_elements,
-                'element_count': len(ui_elements),
-                'processing_time': processing_time,
-                'detection_method': detection_method,
-                'model_info': {
-                    'primary': 'OmniParser v2.0' if 'ui_detector' in self.models else 'YOLOv8',
-                    'gpu': 'T4',
-                    'container_id': os.environ.get('MODAL_TASK_ID', 'unknown')
-                }
-            }
-            
-        except Exception as e:
-            self.logger.error(f"UI detection failed: {e}")
-            return {
-                'success': False,
-                'service': 'isa-vision-ui',
-                'error': str(e),
-                'processing_time': time.time() - start_time
-            }
-    
-    def _omniparser_detection(self, image_np: np.ndarray) -> List[Dict[str, Any]]:
-        """OmniParser-based UI element detection"""
-        # Placeholder implementation
-        # In practice, this would use the actual OmniParser model
-        print("🔍 Using OmniParser for UI detection")
-        
-        # Simulate UI element detection
-        height, width = image_np.shape[:2]
-        ui_elements = []
-        
-        # Mock UI elements (replace with actual OmniParser inference)
-        mock_elements = [
-            {"type": "button", "confidence": 0.95, "bbox": [100, 200, 200, 250]},
-            {"type": "input", "confidence": 0.88, "bbox": [150, 300, 400, 340]},
-            {"type": "text", "confidence": 0.92, "bbox": [50, 100, 300, 130]},
-        ]
-        
-        for i, elem in enumerate(mock_elements):
-            ui_elements.append({
-                'id': f'ui_{i}',
-                'type': elem['type'],
-                'content': f"{elem['type']}_{i}",
-                'center': [
-                    (elem['bbox'][0] + elem['bbox'][2]) // 2,
-                    (elem['bbox'][1] + elem['bbox'][3]) // 2
-                ],
-                'bbox': elem['bbox'],
-                'confidence': elem['confidence'],
-                'interactable': elem['type'] in ['button', 'input', 'link']
-            })
-        
-        return ui_elements
-    
-    def _yolo_detection(self, image_np: np.ndarray) -> List[Dict[str, Any]]:
-        """YOLO-based object detection for UI elements"""
-        model = self.models['detector']
-        results = model(image_np, verbose=False)
-        
-        ui_elements = []
-        
-        if results and results[0].boxes is not None:
-            boxes = results[0].boxes.xyxy.cpu().numpy()
-            confidences = results[0].boxes.conf.cpu().numpy()
-            
-            for i, (box, conf) in enumerate(zip(boxes, confidences)):
-                if conf > 0.3:  # Confidence threshold
-                    x1, y1, x2, y2 = map(int, box)
-                    
-                    ui_elements.append({
-                        'id': f'yolo_{i}',
-                        'type': 'detected_object',
-                        'content': f'object_{i}',
-                        'center': [(x1+x2)//2, (y1+y2)//2],
-                        'bbox': [x1, y1, x2, y2],
-                        'confidence': float(conf),
-                        'interactable': True  # Assume detected objects are interactable
-                    })
-        
-        return ui_elements
-    
-    def _opencv_fallback(self, image_np: np.ndarray) -> List[Dict[str, Any]]:
-        """OpenCV-based fallback detection"""
-        import cv2
-        
-        # Convert to grayscale
-        gray = cv2.cvtColor(image_np, cv2.COLOR_RGB2GRAY)
-        
-        # Edge detection
-        edges = cv2.Canny(gray, 50, 150)
-        
-        # Find contours
-        contours, _ = cv2.findContours(edges, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
-        
-        ui_elements = []
-        for i, contour in enumerate(contours[:10]):  # Limit to 10 largest
-            area = cv2.contourArea(contour)
-            if area > 500:  # Minimum area threshold
-                x, y, w, h = cv2.boundingRect(contour)
-                
-                ui_elements.append({
-                    'id': f'cv_{i}',
-                    'type': 'contour_element',
-                    'content': f'contour_{i}',
-                    'center': [x+w//2, y+h//2],
-                    'bbox': [x, y, x+w, y+h],
-                    'confidence': 0.7,
-                    'interactable': True
-                })
-        
-        return ui_elements
-    
-    @modal.method()
-    def health_check(self) -> Dict[str, Any]:
-        """Health check endpoint"""
-        return {
-            'status': 'healthy',
-            'service': 'isa-vision-ui',
-            'models_loaded': list(self.models.keys()),
-            'timestamp': time.time(),
-            'gpu': 'T4'
-        }
-    
-    def _decode_image(self, image_b64: str) -> Image.Image:
-        """Decode base64 image"""
-        if image_b64.startswith('data:image'):
-            image_b64 = image_b64.split(',')[1]
-        
-        image_data = base64.b64decode(image_b64)
-        return Image.open(io.BytesIO(image_data)).convert('RGB')
-
-# Auto-registration function
-@app.function()
-async def register_service():
-    """Auto-register this service in the model registry"""
-    try:
-        import sys
-        from pathlib import Path
-        
-        # Add project root to path for imports
-        project_root = Path(__file__).parent.parent.parent.parent
-        sys.path.insert(0, str(project_root))
-        
-        try:
-            from isa_model.core.model_manager import ModelManager
-            from isa_model.core.model_repo import ModelType, ModelCapability
-        except ImportError:
-            # Fallback if import fails in Modal environment
-            print("⚠️ Could not import model manager - registration skipped")
-            return {"success": False, "error": "Model manager not available"}
-        
-        # Use ModelManager to register this service
-        model_manager = ModelManager()
-        
-        # Register the service in the registry
-        success = model_manager.registry.register_model(
-            model_id="omniparser-ui-detection-service",
-            model_type=ModelType.VISION,
-            capabilities=[
-                ModelCapability.UI_DETECTION,
-                ModelCapability.IMAGE_ANALYSIS,
-                ModelCapability.IMAGE_UNDERSTANDING
-            ],
-            metadata={
-                "description": "UI element detection service using OmniParser v2.0",
-                "service_name": "isa-vision-ui",
-                "service_type": "modal",
-                "deployment_type": "modal",
-                "endpoint": "https://isa-vision-ui.modal.run",
-                "underlying_model": "microsoft/OmniParser-v2.0",
-                "fallback_model": "ultralytics/yolov8",
-                "gpu_requirement": "T4",
-                "memory_mb": 16384,
-                "auto_registered": True,
-                "registered_by": "isa_vision_ui_service.py",
-                "is_service": True
-            }
-        )
-        
-        if success:
-            print("✅ UI service auto-registered successfully")
-        else:
-            print("⚠️ UI service registration failed")
-            
-        return {"success": success}
-        
-    except Exception as e:
-        print(f"❌ Auto-registration error: {e}")
-        return {"success": False, "error": str(e)}
-
-# Deployment script
-@app.function()
-def deploy_info():
-    """Deployment information"""
-    return {
-        "service": "ISA Vision UI Detection",
-        "model": "microsoft/OmniParser-v2.0 + ultralytics/yolov8 (fallback)",
-        "gpu_requirement": "T4",
-        "memory_requirement": "16GB",
-        "deploy_command": "modal deploy isa_vision_ui_service.py"
-    }
-
-# Quick deployment function
-@app.function()
-def deploy_service():
-    """Deploy this service instantly"""
-    import subprocess
-    import os
-    
-    print("🚀 Deploying ISA Vision UI Service...")
-    try:
-        # Get the current file path
-        current_file = __file__
-        
-        # Run modal deploy command
-        result = subprocess.run(
-            ["modal", "deploy", current_file],
-            capture_output=True,
-            text=True,
-            check=True
-        )
-        
-        print("✅ Deployment completed successfully!")
-        print(f"📝 Output: {result.stdout}")
-        return {"success": True, "output": result.stdout}
-        
-    except subprocess.CalledProcessError as e:
-        print(f"❌ Deployment failed: {e}")
-        print(f"📝 Error: {e.stderr}")
-        return {"success": False, "error": str(e), "stderr": e.stderr}
-
-if __name__ == "__main__":
-    print("🚀 ISA Vision UI Service - Modal Deployment")
-    print("Deploy with: modal deploy isa_vision_ui_service.py")
-    print("Or call: modal run isa_vision_ui_service.py::deploy_service")
-    print("Note: Uses OmniParser v2.0 with YOLOv8 fallback")
-    print("\n📝 Service will auto-register in model registry upon deployment")