isa-model 0.3.4__py3-none-any.whl → 0.3.5__py3-none-any.whl

isa_model/deployment/cloud/modal/isa_vision_ui_service.py

@@ -0,0 +1,305 @@
+"""
+ISA Vision UI Service
+
+Specialized service for UI element detection using OmniParser v2.0
+Fallback to YOLOv8 for general object detection
+"""
+
+import modal
+import torch
+import base64
+import io
+import numpy as np
+from PIL import Image
+from typing import Dict, List, Optional, Any
+import time
+import json
+import os
+import logging
+
+# Define Modal application
+app = modal.App("isa-vision-ui")
+
+# Download UI detection models
+def download_ui_models():
+    """Download UI detection models"""
+    from huggingface_hub import snapshot_download
+    
+    print("📦 Downloading UI detection models...")
+    os.makedirs("/models", exist_ok=True)
+    
+    # Download OmniParser v2.0
+    try:
+        snapshot_download(
+            repo_id="microsoft/OmniParser-v2.0",
+            local_dir="/models/omniparser-v2",
+            allow_patterns=["**/*.pt", "**/*.pth", "**/*.bin", "**/*.json", "**/*.safetensors"]
+        )
+        print("✅ OmniParser v2.0 downloaded")
+    except Exception as e:
+        print(f"⚠️ OmniParser v2.0 download failed: {e}")
+    
+    # Download YOLOv8 (fallback)
+    try:
+        from ultralytics import YOLO
+        model = YOLO('yolov8n.pt')
+        print("✅ YOLOv8 fallback model downloaded")
+    except Exception as e:
+        print(f"⚠️ YOLOv8 download failed: {e}")
+    
+    print("📦 UI models download completed")
+
+# Define Modal container image
+image = (
+    modal.Image.debian_slim(python_version="3.11")
+    .pip_install([
+        # Core AI libraries
+        "torch>=2.0.0",
+        "torchvision", 
+        "transformers>=4.35.0",
+        "ultralytics>=8.0.43",
+        "huggingface_hub",
+        "accelerate",
+        
+        # Image processing
+        "pillow>=10.0.1",
+        "opencv-python-headless",
+        "numpy>=1.24.3",
+        
+        # HTTP libraries
+        "httpx>=0.26.0",
+        "requests",
+        
+        # Utilities
+        "pydantic>=2.0.0",
+        "python-dotenv",
+    ])
+    .run_function(download_ui_models)
+    .env({"TRANSFORMERS_CACHE": "/models"})
+)
+
+# UI Detection Service
+@app.cls(
+    gpu="T4",
+    image=image,
+    memory=16384,  # 16GB RAM
+    timeout=1800,  # 30 minutes
+    scaledown_window=300,  # 5 minutes idle timeout
+    min_containers=0,  # Scale to zero to save costs
+)
+class UIDetectionService:
+    """
+    UI Element Detection Service
+    
+    Provides fast UI element detection using OmniParser v2.0
+    Falls back to YOLOv8 for general object detection
+    """
+    
+    def __init__(self):
+        self.models = {}
+        self.logger = logging.getLogger(__name__)
+        
+    @modal.enter()
+    def load_models(self):
+        """Load UI detection models on container startup"""
+        print("🚀 Loading UI detection models...")
+        start_time = time.time()
+        
+        # Try to load OmniParser first
+        try:
+            self._load_omniparser()
+        except Exception as e:
+            print(f"⚠️ OmniParser failed to load: {e}")
+            # Fall back to YOLOv8
+            self._load_yolo_fallback()
+        
+        load_time = time.time() - start_time
+        print(f"✅ UI detection models loaded in {load_time:.2f}s")
+        
+    def _load_omniparser(self):
+        """Load OmniParser model"""
+        # Placeholder for actual OmniParser loading
+        # In practice, you would load the actual OmniParser model here
+        print("📱 Loading OmniParser v2.0...")
+        self.models['ui_detector'] = "omniparser_placeholder"
+        print("✅ OmniParser v2.0 loaded")
+        
+    def _load_yolo_fallback(self):
+        """Load YOLOv8 as fallback"""
+        from ultralytics import YOLO
+        
+        print("🔄 Loading YOLOv8 fallback...")
+        yolo_model = YOLO('yolov8n.pt')
+        self.models['detector'] = yolo_model
+        print("✅ YOLOv8 fallback loaded")
+    
+    @modal.method()
+    def detect_ui_elements(self, image_b64: str, detection_type: str = "ui") -> Dict[str, Any]:
+        """
+        Detect UI elements in image
+        
+        Args:
+            image_b64: Base64 encoded image
+            detection_type: Type of detection ("ui" or "general")
+            
+        Returns:
+            Detection results with UI elements
+        """
+        start_time = time.time()
+        
+        try:
+            # Decode image
+            image = self._decode_image(image_b64)
+            image_np = np.array(image)
+            
+            # Perform detection based on available models
+            if 'ui_detector' in self.models:
+                ui_elements = self._omniparser_detection(image_np)
+                detection_method = "omniparser"
+            elif 'detector' in self.models:
+                ui_elements = self._yolo_detection(image_np)
+                detection_method = "yolo_fallback"
+            else:
+                ui_elements = self._opencv_fallback(image_np)
+                detection_method = "opencv_fallback"
+            
+            processing_time = time.time() - start_time
+            
+            return {
+                'success': True,
+                'service': 'isa-vision-ui',
+                'ui_elements': ui_elements,
+                'element_count': len(ui_elements),
+                'processing_time': processing_time,
+                'detection_method': detection_method,
+                'model_info': {
+                    'primary': 'OmniParser v2.0' if 'ui_detector' in self.models else 'YOLOv8',
+                    'gpu': 'T4',
+                    'container_id': os.environ.get('MODAL_TASK_ID', 'unknown')
+                }
+            }
+            
+        except Exception as e:
+            self.logger.error(f"UI detection failed: {e}")
+            return {
+                'success': False,
+                'service': 'isa-vision-ui',
+                'error': str(e),
+                'processing_time': time.time() - start_time
+            }
+    
+    def _omniparser_detection(self, image_np: np.ndarray) -> List[Dict[str, Any]]:
+        """OmniParser-based UI element detection"""
+        # Placeholder implementation
+        # In practice, this would use the actual OmniParser model
+        print("🔍 Using OmniParser for UI detection")
+        
+        # Simulate UI element detection
+        height, width = image_np.shape[:2]
+        ui_elements = []
+        
+        # Mock UI elements (replace with actual OmniParser inference)
+        mock_elements = [
+            {"type": "button", "confidence": 0.95, "bbox": [100, 200, 200, 250]},
+            {"type": "input", "confidence": 0.88, "bbox": [150, 300, 400, 340]},
+            {"type": "text", "confidence": 0.92, "bbox": [50, 100, 300, 130]},
+        ]
+        
+        for i, elem in enumerate(mock_elements):
+            ui_elements.append({
+                'id': f'ui_{i}',
+                'type': elem['type'],
+                'content': f"{elem['type']}_{i}",
+                'center': [
+                    (elem['bbox'][0] + elem['bbox'][2]) // 2,
+                    (elem['bbox'][1] + elem['bbox'][3]) // 2
+                ],
+                'bbox': elem['bbox'],
+                'confidence': elem['confidence'],
+                'interactable': elem['type'] in ['button', 'input', 'link']
+            })
+        
+        return ui_elements
+    
+    def _yolo_detection(self, image_np: np.ndarray) -> List[Dict[str, Any]]:
+        """YOLO-based object detection for UI elements"""
+        model = self.models['detector']
+        results = model(image_np, verbose=False)
+        
+        ui_elements = []
+        
+        if results and results[0].boxes is not None:
+            boxes = results[0].boxes.xyxy.cpu().numpy()
+            confidences = results[0].boxes.conf.cpu().numpy()
+            
+            for i, (box, conf) in enumerate(zip(boxes, confidences)):
+                if conf > 0.3:  # Confidence threshold
+                    x1, y1, x2, y2 = map(int, box)
+                    
+                    ui_elements.append({
+                        'id': f'yolo_{i}',
+                        'type': 'detected_object',
+                        'content': f'object_{i}',
+                        'center': [(x1+x2)//2, (y1+y2)//2],
+                        'bbox': [x1, y1, x2, y2],
+                        'confidence': float(conf),
+                        'interactable': True  # Assume detected objects are interactable
+                    })
+        
+        return ui_elements
+    
+    def _opencv_fallback(self, image_np: np.ndarray) -> List[Dict[str, Any]]:
+        """OpenCV-based fallback detection"""
+        import cv2
+        
+        # Convert to grayscale
+        gray = cv2.cvtColor(image_np, cv2.COLOR_RGB2GRAY)
+        
+        # Edge detection
+        edges = cv2.Canny(gray, 50, 150)
+        
+        # Find contours
+        contours, _ = cv2.findContours(edges, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+        
+        ui_elements = []
+        for i, contour in enumerate(contours[:10]):  # Limit to 10 largest
+            area = cv2.contourArea(contour)
+            if area > 500:  # Minimum area threshold
+                x, y, w, h = cv2.boundingRect(contour)
+                
+                ui_elements.append({
+                    'id': f'cv_{i}',
+                    'type': 'contour_element',
+                    'content': f'contour_{i}',
+                    'center': [x+w//2, y+h//2],
+                    'bbox': [x, y, x+w, y+h],
+                    'confidence': 0.7,
+                    'interactable': True
+                })
+        
+        return ui_elements
+    
+    @modal.method()
+    def health_check(self) -> Dict[str, Any]:
+        """Health check endpoint"""
+        return {
+            'status': 'healthy',
+            'service': 'isa-vision-ui',
+            'models_loaded': list(self.models.keys()),
+            'timestamp': time.time(),
+            'gpu': 'T4'
+        }
+    
+    def _decode_image(self, image_b64: str) -> Image.Image:
+        """Decode base64 image"""
+        if image_b64.startswith('data:image'):
+            image_b64 = image_b64.split(',')[1]
+        
+        image_data = base64.b64decode(image_b64)
+        return Image.open(io.BytesIO(image_data)).convert('RGB')
+
+# Warmup function removed to save costs
+
+if __name__ == "__main__":
+    print("🚀 ISA Vision UI Service - Modal Deployment")
+    print("Deploy with: modal deploy isa_vision_ui_service.py")

isa_model/inference/ai_factory.py

@@ -13,6 +13,7 @@ from isa_model.inference.services.base_service import BaseService
 from isa_model.inference.base import ModelType
 from isa_model.inference.services.vision.base_vision_service import BaseVisionService
 from isa_model.inference.services.vision.base_image_gen_service import BaseImageGenService
+from isa_model.inference.services.stacked import UIAnalysisService, BaseStackedService, DocAnalysisStackedService, FluxProfessionalService
 
 if TYPE_CHECKING:
     from isa_model.inference.services.audio.base_stt_service import BaseSTTService
@@ -55,6 +56,12 @@ class AIFactory:
             # Register Replicate services
             self._register_replicate_services()
             
+            # Register ISA Modal services
+            self._register_isa_services()
+            
+            # Register YYDS services
+            self._register_yyds_services()
+            
             logger.info("AI Factory initialized with centralized provider API key management")
             
         except Exception as e:
@@ -105,10 +112,16 @@ class AIFactory:
         try:
             from isa_model.inference.providers.replicate_provider import ReplicateProvider
             from isa_model.inference.services.vision.replicate_image_gen_service import ReplicateImageGenService
+            from isa_model.inference.services.vision.replicate_vision_service import ReplicateVisionService
             from isa_model.inference.services.audio.replicate_tts_service import ReplicateTTSService
             
             self.register_provider('replicate', ReplicateProvider)
-            self.register_service('replicate', ModelType.VISION, ReplicateImageGenService)
+            # Register vision service for general vision tasks
+            self.register_service('replicate', ModelType.VISION, ReplicateVisionService)
+            # Register image generation service for FLUX, ControlNet, LoRA, Upscaling
+            # Note: Using VISION type as IMAGE_GEN is not defined in ModelType
+            # ReplicateImageGenService will be accessed through get_img() methods
+            # Register audio service
             self.register_service('replicate', ModelType.AUDIO, ReplicateTTSService)
             
             logger.info("Replicate services registered successfully")
@@ -116,6 +129,34 @@ class AIFactory:
         except ImportError as e:
             logger.warning(f"Replicate services not available: {e}")
     
+    def _register_isa_services(self):
+        """Register ISA Modal provider and services"""
+        try:
+            from isa_model.inference.services.vision.isA_vision_service import ISAVisionService
+            from isa_model.inference.providers.modal_provider import ModalProvider
+            
+            self.register_provider('modal', ModalProvider)
+            self.register_service('modal', ModelType.VISION, ISAVisionService)
+            
+            logger.info("ISA Modal services registered successfully")
+            
+        except ImportError as e:
+            logger.warning(f"ISA Modal services not available: {e}")
+    
+    def _register_yyds_services(self):
+        """Register YYDS provider and services"""
+        try:
+            from isa_model.inference.providers.yyds_provider import YydsProvider
+            from isa_model.inference.services.llm.yyds_llm_service import YydsLLMService
+            
+            self.register_provider('yyds', YydsProvider)
+            self.register_service('yyds', ModelType.LLM, YydsLLMService)
+            
+            logger.info("YYDS services registered successfully")
+            
+        except ImportError as e:
+            logger.warning(f"YYDS services not available: {e}")
+    
     def register_provider(self, name: str, provider_class: Type[BaseProvider]) -> None:
         """Register an AI provider"""
         self._providers[name] = provider_class
@@ -164,7 +205,7 @@ class AIFactory:
         Get a LLM service instance with automatic defaults
         
         Args:
-            model_name: Name of the model to use (defaults: OpenAI="gpt-4.1-nano", Ollama="llama3.2:3b")
+            model_name: Name of the model to use (defaults: OpenAI="gpt-4.1-nano", Ollama="llama3.2:3b", YYDS="claude-sonnet-4-20250514")
             provider: Provider name (defaults to 'openai' for production, 'ollama' for dev)
             config: Optional configuration dictionary (auto-loads from .env if not provided)
                    Can include: streaming=True/False, temperature, max_tokens, etc.
@@ -179,6 +220,9 @@ class AIFactory:
         elif provider == "ollama":
             final_model_name = model_name or "llama3.2:3b-instruct-fp16"
             final_provider = provider
+        elif provider == "yyds":
+            final_model_name = model_name or "claude-sonnet-4-20250514"
+            final_provider = provider
         else:
             # Default provider selection - OpenAI with cheapest model
             final_provider = provider or "openai"
@@ -249,27 +293,70 @@ class AIFactory:
         
         return cast(BaseVisionService, self.create_service(final_provider, ModelType.VISION, final_model_name, config))
     
-    def get_image_generation_service(self, model_name: Optional[str] = None, provider: Optional[str] = None,
-                                   config: Optional[Dict[str, Any]] = None) -> 'BaseImageGenService':
+    def get_image_gen(self, model_name: Optional[str] = None, provider: Optional[str] = None,
+                     config: Optional[Dict[str, Any]] = None) -> 'BaseImageGenService':
         """
         Get an image generation service instance with automatic defaults
         
         Args:
-            model_name: Name of the model to use (defaults: "black-forest-labs/flux-schnell" for production)
+            model_name: Name of the model to use. Supports:
+                - FLUX models: "flux-pro", "flux-schnell", "flux-dev"
+                - ControlNet: "flux-controlnet", "xlabs-ai/flux-dev-controlnet"  
+                - LoRA: "flux-lora", "flux-dev-lora"
+                - InstantID: "instant-id", "zsxkib/instant-id"
+                - Character: "consistent-character", "fofr/consistent-character"
+                - Upscaling: "ultimate-upscaler", "ultimate-sd-upscale"
+                - Detail: "adetailer"
             provider: Provider name (defaults to 'replicate')
-            config: Optional configuration dictionary (auto-loads from .env if not provided)
+            config: Optional configuration dictionary
             
         Returns:
-            Image generation service instance
+            Image generation service instance with FLUX, ControlNet, LoRA, InstantID, Upscaling support
         """
         # Set defaults based on provider
         final_provider = provider or "replicate"
-        if final_provider == "replicate":
-            final_model_name = model_name or "black-forest-labs/flux-schnell"
+        
+        # Default model selection
+        if not model_name:
+            final_model_name = "black-forest-labs/flux-schnell"
         else:
-            final_model_name = model_name or "black-forest-labs/flux-schnell"
+            # Map short names to full Replicate model names
+            model_mapping = {
+                "flux-pro": "black-forest-labs/flux-pro",
+                "flux-schnell": "black-forest-labs/flux-schnell", 
+                "flux-dev": "black-forest-labs/flux-dev",
+                "flux-controlnet": "xlabs-ai/flux-dev-controlnet",
+                "flux-lora": "xlabs-ai/flux-lora",
+                "instant-id": "zsxkib/instant-id",
+                "consistent-character": "fofr/consistent-character", 
+                "ultimate-upscaler": "philz1337x/clarity-upscaler",
+                "ultimate-sd-upscale": "philz1337x/clarity-upscaler",
+                "adetailer": "sczhou/codeformer"
+            }
+            final_model_name = model_mapping.get(model_name, model_name)
         
-        return cast('BaseImageGenService', self.create_service(final_provider, ModelType.VISION, final_model_name, config))
+        # Create ReplicateImageGenService directly for image generation
+        try:
+            from isa_model.inference.services.vision.replicate_image_gen_service import ReplicateImageGenService
+            from isa_model.inference.providers.replicate_provider import ReplicateProvider
+            
+            # Create provider with config
+            provider_instance = ReplicateProvider(config=config)
+            service = ReplicateImageGenService(provider=provider_instance, model_name=final_model_name)
+            
+            return service
+            
+        except ImportError as e:
+            logger.error(f"Failed to import ReplicateImageGenService: {e}")
+            raise ValueError(f"Image generation service not available: {e}")
+        except Exception as e:
+            logger.error(f"Failed to create image generation service: {e}")
+            raise
+    
+    def get_image_generation_service(self, model_name: Optional[str] = None, provider: Optional[str] = None,
+                                   config: Optional[Dict[str, Any]] = None) -> 'BaseImageGenService':
+        """Alias for get_image_gen() method"""
+        return self.get_image_gen(model_name, provider, config)
     
     def get_img(self, type: str = "t2i", model_name: Optional[str] = None, provider: Optional[str] = None,
                 config: Optional[Dict[str, Any]] = None) -> 'BaseImageGenService':
@@ -310,7 +397,8 @@ class AIFactory:
         else:
             raise ValueError(f"Unknown image generation type: {type}. Use 't2i' or 'i2i'")
         
-        return cast('BaseImageGenService', self.create_service(final_provider, ModelType.VISION, final_model_name, config))
+        # Use the new get_image_gen method
+        return self.get_image_gen(final_model_name, final_provider, config)
     
     def get_audio_service(self, model_name: Optional[str] = None, provider: Optional[str] = None,
                          config: Optional[Dict[str, Any]] = None) -> BaseService:
@@ -436,12 +524,13 @@ class AIFactory:
         Usage:
             llm = AIFactory().get_llm()  # Uses gpt-4.1-nano by default
             llm = AIFactory().get_llm(model_name="llama3.2", provider="ollama")
+            llm = AIFactory().get_llm(provider="yyds")  # Uses claude-sonnet-4-20250514 by default
             llm = AIFactory().get_llm(model_name="gpt-4.1-mini", provider="openai", config={"streaming": True})
         """
         return self.get_llm_service(model_name, provider, config)
     
     def get_embed(self, model_name: Optional[str] = None, provider: Optional[str] = None,
-                     config: Optional[Dict[str, Any]] = None) -> 'BaseEmbedService':
+                     config: Optional[Dict[str, Any]] = None) -> BaseService:
         """
         Get embedding service with automatic defaults
         
@@ -547,4 +636,139 @@ class AIFactory:
             model_name=model_name,
             provider=provider,
             config=config
-        ) 
+        )
+    
+    def get_provider(self, provider_name: str, config: Optional[Dict[str, Any]] = None) -> BaseProvider:
+        """
+        Get a provider instance
+        
+        Args:
+            provider_name: Name of the provider ('openai', 'ollama', 'replicate')
+            config: Optional configuration override
+            
+        Returns:
+            Provider instance
+        """
+        if provider_name not in self._providers:
+            raise ValueError(f"No provider registered for '{provider_name}'")
+        
+        provider_class = self._providers[provider_name]
+        return provider_class(config=config)
+    
+    def get_stacked(
+        self,
+        service_name: str,
+        config: Optional[Dict[str, Any]] = None
+    ) -> BaseStackedService:
+        """
+        Get a stacked service by name with automatic defaults
+        
+        Args:
+            service_name: Name of the stacked service ('ui_analysis', etc.)
+            config: Optional configuration override
+            
+        Returns:
+            Stacked service instance
+            
+        Usage:
+            ui_service = AIFactory().get_stacked("ui_analysis", {"task_type": "search"})
+        """
+        if service_name == "ui_analysis":
+            return UIAnalysisService(self, config)
+        elif service_name == "search_analysis":
+            if config is None:
+                config = {}
+            config["task_type"] = "search"
+            return UIAnalysisService(self, config)
+        elif service_name == "content_analysis":
+            if config is None:
+                config = {}
+            config["task_type"] = "content"
+            return UIAnalysisService(self, config)
+        elif service_name == "navigation_analysis":
+            if config is None:
+                config = {}
+            config["task_type"] = "navigation"
+            return UIAnalysisService(self, config)
+        elif service_name == "doc_analysis":
+            return DocAnalysisStackedService(self, config)
+        elif service_name == "flux_professional":
+            return FluxProfessionalService(self)
+        else:
+            raise ValueError(f"Unknown stacked service: {service_name}. Available: ui_analysis, search_analysis, content_analysis, navigation_analysis, doc_analysis, flux_professional")
+    
+    def get_ui_analysis(
+        self,
+        task_type: str = "login",
+        config: Optional[Dict[str, Any]] = None
+    ) -> UIAnalysisService:
+        """
+        Get UI Analysis service with task-specific configuration
+        
+        Args:
+            task_type: Type of UI task ('login', 'search', 'content', 'navigation')
+            config: Optional configuration override
+        
+        Usage:
+            # For login pages (default)
+            ui_service = AIFactory().get_ui_analysis()
+            
+            # For search pages
+            ui_service = AIFactory().get_ui_analysis(task_type="search")
+            
+            # For content extraction
+            ui_service = AIFactory().get_ui_analysis(task_type="content")
+        """
+        if config is None:
+            config = {}
+        config["task_type"] = task_type
+        return cast(UIAnalysisService, self.get_stacked("ui_analysis", config))
+    
+    def get_doc_analysis(
+        self,
+        config: Optional[Dict[str, Any]] = None
+    ) -> DocAnalysisStackedService:
+        """
+        Get Document Analysis service with 5-step pipeline
+        
+        Args:
+            config: Optional configuration override
+        
+        Usage:
+            # Basic document analysis
+            doc_service = AIFactory().get_doc_analysis()
+            
+            # Analyze a document image
+            result = await doc_service.analyze_document("document.png")
+            
+            # Get structured data ready for business mapping
+            structured_data = result["final_output"]["final_structured_data"]
+        """
+        return cast(DocAnalysisStackedService, self.get_stacked("doc_analysis", config))
+    
+    def get_flux_professional(
+        self,
+        config: Optional[Dict[str, Any]] = None
+    ) -> FluxProfessionalService:
+        """
+        Get FLUX Professional Pipeline service for multi-stage image generation
+        
+        Args:
+            config: Optional configuration override
+        
+        Usage:
+            # Basic professional image generation
+            flux_service = AIFactory().get_flux_professional()
+            
+            # Generate professional image with character consistency
+            result = await flux_service.invoke({
+                "prompt": "portrait of a warrior in fantasy armor",
+                "face_image": "reference_face.jpg",  # For character consistency
+                "lora_style": "realism",
+                "upscale_factor": 4
+            })
+            
+            # Get final high-quality image
+            final_image_url = result["final_output"]["image_url"]
+        """
+        return cast(FluxProfessionalService, self.get_stacked("flux_professional", config))