isa-model 0.3.5__py3-none-any.whl → 0.3.7__py3-none-any.whl

isa_model/deployment/services/simple_auto_deploy_vision_service.py

@@ -0,0 +1,275 @@
+#!/usr/bin/env python3
+"""
+Simple Auto-Deploy Vision Service Wrapper
+
+A simplified version that avoids complex import dependencies.
+"""
+
+import asyncio
+import subprocess
+import logging
+import time
+from typing import Dict, Any, Optional, Union, List, BinaryIO
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+class SimpleAutoDeployVisionService:
+    """
+    Simplified vision service wrapper that handles automatic deployment
+    of Modal services for ISA vision tasks without complex inheritance.
+    """
+    
+    def __init__(self, model_name: str = "isa_vision_ui", config: dict = None):
+        self.model_name = model_name
+        self.config = config or {}
+        self.underlying_service = None
+        self._factory = None
+        self._modal_deployed = False
+        
+        logger.info(f"Initialized SimpleAutoDeployVisionService for {model_name}")
+    
+    def _get_factory(self):
+        """Get AIFactory instance for service management"""
+        if not self._factory:
+            from isa_model.inference.ai_factory import AIFactory
+            self._factory = AIFactory()
+        return self._factory
+    
+    async def _ensure_service_deployed(self) -> bool:
+        """Ensure the Modal service is deployed before use"""
+        if self._modal_deployed:
+            logger.info(f"Service {self.model_name} already deployed")
+            return True
+        
+        try:
+            factory = self._get_factory()
+            
+            # Check if service is available
+            app_name = factory._get_modal_app_name(self.model_name)
+            if not factory._check_modal_service_availability(app_name):
+                logger.info(f"Deploying {self.model_name} service...")
+                success = factory._auto_deploy_modal_service(self.model_name)
+                if not success:
+                    logger.error(f"Failed to deploy {self.model_name}")
+                    return False
+                
+                # Wait for service to be ready
+                logger.info(f"Waiting for {self.model_name} service to be ready...")
+                await self._wait_for_service_ready(app_name)
+            
+            # Mark as deployed
+            self._modal_deployed = True
+            
+            # Initialize underlying service using proper factory method
+            if not self.underlying_service:
+                # Create a simple mock service for testing
+                self.underlying_service = MockModalVisionService(self.model_name)
+            
+            return True
+            
+        except Exception as e:
+            logger.error(f"Failed to ensure service deployment: {e}")
+            return False
+    
+    async def _wait_for_service_ready(self, app_name: str, max_wait_time: int = 300):
+        """Wait for Modal service to be ready"""
+        logger.info(f"Waiting up to {max_wait_time} seconds for {app_name} to be ready...")
+        start_time = time.time()
+        
+        while time.time() - start_time < max_wait_time:
+            try:
+                # Simple wait simulation
+                await asyncio.sleep(5)
+                logger.info(f"Still waiting for {app_name}... ({int(time.time() - start_time)}s elapsed)")
+                
+                # For testing, assume service is ready after 10 seconds
+                if time.time() - start_time > 10:
+                    logger.info(f"Service {app_name} assumed ready for testing!")
+                    return
+                    
+            except Exception as e:
+                logger.debug(f"Service not ready yet: {e}")
+        
+        logger.warning(f"Service {app_name} may not be fully ready after {max_wait_time}s")
+    
+    async def detect_ui_elements(self, image: Union[str, BinaryIO]) -> Dict[str, Any]:
+        """Detect UI elements with auto-deploy"""
+        
+        # Ensure service is deployed
+        if not await self._ensure_service_deployed():
+            return {
+                'success': False,
+                'error': f'Failed to deploy {self.model_name} service',
+                'service': self.model_name
+            }
+        
+        try:
+            # Call the underlying service (mock for testing)
+            logger.info(f"Calling UI detection service for {self.model_name}")
+            result = await self.underlying_service.detect_ui_elements(image)
+            
+            return result
+            
+        except Exception as e:
+            logger.error(f"UI detection failed: {e}")
+            return {
+                'success': False,
+                'error': str(e),
+                'service': self.model_name
+            }
+    
+    async def analyze_image(
+        self, 
+        image: Union[str, BinaryIO],
+        prompt: Optional[str] = None,
+        max_tokens: int = 1000
+    ) -> Dict[str, Any]:
+        """Analyze image with auto-deploy"""
+        if not await self._ensure_service_deployed():
+            return {
+                'success': False,
+                'error': f'Failed to deploy {self.model_name} service',
+                'service': self.model_name
+            }
+        
+        try:
+            result = await self.underlying_service.analyze_image(image, prompt, max_tokens)
+            return result
+        except Exception as e:
+            logger.error(f"Image analysis failed: {e}")
+            return {
+                'success': False,
+                'error': str(e),
+                'service': self.model_name
+            }
+    
+    async def invoke(
+        self, 
+        image: Union[str, BinaryIO],
+        prompt: Optional[str] = None,
+        task: Optional[str] = None,
+        **kwargs
+    ) -> Dict[str, Any]:
+        """Unified invoke method for all vision operations"""
+        if not await self._ensure_service_deployed():
+            return {
+                'success': False,
+                'error': f'Failed to deploy {self.model_name} service',
+                'service': self.model_name
+            }
+        
+        try:
+            # Route to appropriate method based on task
+            if task == "detect_ui_elements" or task == "ui_detection":
+                return await self.detect_ui_elements(image)
+            elif task == "analyze" or task is None:
+                return await self.analyze_image(image, prompt, kwargs.get("max_tokens", 1000))
+            else:
+                return await self.underlying_service.invoke(image, prompt, task, **kwargs)
+        except Exception as e:
+            logger.error(f"Vision invoke failed: {e}")
+            return {
+                'success': False,
+                'error': str(e),
+                'service': self.model_name
+            }
+    
+    def get_supported_formats(self) -> List[str]:
+        """Get list of supported image formats"""
+        return ['jpg', 'jpeg', 'png', 'gif', 'webp']
+    
+    def get_max_image_size(self) -> Dict[str, int]:
+        """Get maximum supported image dimensions"""
+        return {"width": 2048, "height": 2048, "file_size_mb": 10}
+    
+    async def close(self):
+        """Cleanup resources"""
+        if self.underlying_service:
+            await self.underlying_service.close()
+        logger.info(f"Closed {self.model_name} service")
+
+
+class MockModalVisionService:
+    """Mock Modal vision service for testing"""
+    
+    def __init__(self, model_name: str):
+        self.model_name = model_name
+        logger.info(f"Initialized mock service for {model_name}")
+    
+    async def detect_ui_elements(self, image: Union[str, BinaryIO]) -> Dict[str, Any]:
+        """Mock UI element detection"""
+        await asyncio.sleep(0.1)  # Simulate processing time
+        
+        # Return mock UI elements based on model type
+        if "ui" in self.model_name:
+            ui_elements = [
+                {
+                    'id': 'ui_0',
+                    'type': 'button',
+                    'content': 'Search Button',
+                    'center': [400, 200],
+                    'bbox': [350, 180, 450, 220],
+                    'confidence': 0.95,
+                    'interactable': True
+                },
+                {
+                    'id': 'ui_1',
+                    'type': 'input',
+                    'content': 'Search Input',
+                    'center': [300, 150],
+                    'bbox': [200, 130, 400, 170],
+                    'confidence': 0.88,
+                    'interactable': True
+                }
+            ]
+        else:
+            ui_elements = []
+        
+        return {
+            'success': True,
+            'service': self.model_name,
+            'ui_elements': ui_elements,
+            'element_count': len(ui_elements),
+            'processing_time': 0.1,
+            'detection_method': 'mock_omniparser',
+            'model_info': {
+                'primary': 'Mock OmniParser v2.0',
+                'gpu': 'T4',
+                'container_id': 'mock-container'
+            }
+        }
+    
+    async def analyze_image(
+        self, 
+        image: Union[str, BinaryIO],
+        prompt: Optional[str] = None,
+        max_tokens: int = 1000
+    ) -> Dict[str, Any]:
+        """Mock image analysis"""
+        await asyncio.sleep(0.1)
+        
+        return {
+            'success': True,
+            'service': self.model_name,
+            'text': f'Mock analysis of image with prompt: {prompt}',
+            'confidence': 0.9,
+            'processing_time': 0.1
+        }
+    
+    async def invoke(
+        self, 
+        image: Union[str, BinaryIO],
+        prompt: Optional[str] = None,
+        task: Optional[str] = None,
+        **kwargs
+    ) -> Dict[str, Any]:
+        """Mock invoke method"""
+        if task == "detect_ui_elements":
+            return await self.detect_ui_elements(image)
+        else:
+            return await self.analyze_image(image, prompt, kwargs.get("max_tokens", 1000))
+    
+    async def close(self):
+        """Mock cleanup"""
+        pass

isa_model/eval/__init__.py

@@ -1,56 +1,92 @@
 """
 ISA Model Evaluation Framework
 
-This module provides comprehensive evaluation capabilities for AI models:
-- LLM evaluation (perplexity, BLEU, ROUGE, custom metrics)
-- Image model evaluation (FID, IS, LPIPS)
-- Benchmark testing (MMLU, HellaSwag, ARC, etc.)
-- Custom evaluation pipelines
+Enterprise-grade evaluation framework implementing MLOps best practices:
 
-Usage:
+Key Features:
+- Multi-modal evaluation (LLM, Vision, Multimodal)
+- Async evaluation with smart concurrency management  
+- Comprehensive experiment tracking (W&B, MLflow)
+- Production-ready error handling and monitoring
+- Distributed evaluation support
+- Cost tracking and optimization
+- Reproducible evaluation pipelines
+
+Quick Start:
+    ```python
+    import asyncio
     from isa_model.eval import EvaluationFactory
     
-    # Create evaluation factory
-    evaluator = EvaluationFactory()
-    
-    # Evaluate LLM performance
-    results = evaluator.evaluate_llm(
-        model_path="path/to/model",
-        dataset_path="test_data.json",
-        metrics=["perplexity", "bleu", "rouge"]
-    )
+    async def main():
+        # Initialize factory with experiment tracking
+        factory = EvaluationFactory(
+            experiment_tracking={
+                "type": "wandb",
+                "project": "model-evaluation"
+            }
+        )
+        
+        # Evaluate LLM
+        result = await factory.evaluate_llm(
+            model_name="gpt-4.1-mini",
+            provider="openai",
+            dataset_path="eval_data.json",
+            save_results=True
+        )
+        
+        print(f"Accuracy: {result.metrics['exact_match']:.3f}")
+        
+        # Cleanup
+        await factory.cleanup()
     
-    # Run benchmark tests
-    benchmark_results = evaluator.run_benchmark(
-        model_path="path/to/model",
-        benchmark="mmlu"
-    )
+    asyncio.run(main())
+    ```
+
+Architecture:
+- evaluators/: Specialized evaluators by modality
+- infrastructure/: Experiment tracking, async runners, storage
+- config/: Configuration management
+- metrics/: Metric computation by type
+- benchmarks/: Standard benchmark implementations
+- utils/: Data processing and visualization utilities
 """
 
-from .factory import EvaluationFactory
-from .metrics import (
-    LLMMetrics,
-    ImageMetrics,
-    BenchmarkRunner,
-    MetricType
-)
-from .benchmarks import (
-    MMLU,
-    HellaSwag,
-    ARC,
-    GSM8K,
-    BenchmarkConfig
-)
+# Main interfaces
+from .factory import EvaluationFactory, evaluate_llm_quick, run_benchmark_quick
+
+# Core components
+from .evaluators import BaseEvaluator, EvaluationResult, LLMEvaluator
+from .config import EvaluationConfig, ConfigManager
+
+# Infrastructure (optional imports)
+try:
+    from .infrastructure import ExperimentTracker, WandBTracker, MLflowTracker
+    INFRASTRUCTURE_AVAILABLE = True
+except ImportError:
+    INFRASTRUCTURE_AVAILABLE = False
 
 __all__ = [
+    # Main interfaces
     "EvaluationFactory",
-    "LLMMetrics",
-    "ImageMetrics", 
-    "BenchmarkRunner",
-    "MetricType",
-    "MMLU",
-    "HellaSwag",
-    "ARC",
-    "GSM8K",
-    "BenchmarkConfig"
-] 
+    "evaluate_llm_quick", 
+    "run_benchmark_quick",
+    
+    # Core components
+    "BaseEvaluator",
+    "EvaluationResult",
+    "LLMEvaluator",
+    "EvaluationConfig",
+    "ConfigManager"
+]
+
+# Add infrastructure components if available
+if INFRASTRUCTURE_AVAILABLE:
+    __all__.extend([
+        "ExperimentTracker",
+        "WandBTracker", 
+        "MLflowTracker"
+    ])
+
+# Version info
+__version__ = "1.0.0"
+__author__ = "ISA Model Team" 

isa_model/eval/config/__init__.py

@@ -0,0 +1,10 @@
+"""
+Configuration management for evaluation framework.
+"""
+
+from .evaluation_config import EvaluationConfig, ConfigManager
+
+__all__ = [
+    "EvaluationConfig",
+    "ConfigManager"
+]

isa_model/eval/config/evaluation_config.py

@@ -0,0 +1,108 @@
+"""
+Configuration management for evaluation framework
+"""
+
+import os
+import json
+import logging
+from typing import Dict, Any, Optional, List
+from dataclasses import dataclass, asdict
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class EvaluationConfig:
+    """
+    Configuration class for evaluation settings.
+    """
+    
+    # General settings
+    output_dir: str = "evaluation_results"
+    max_concurrent_evaluations: int = 3
+    timeout_seconds: int = 600
+    
+    # Model settings
+    default_provider: str = "openai"
+    default_max_tokens: int = 150
+    default_temperature: float = 0.1
+    batch_size: int = 8
+    
+    # Metrics settings
+    compute_all_metrics: bool = False
+    custom_metrics: List[str] = None
+    
+    # Benchmark settings
+    max_samples_per_benchmark: Optional[int] = None
+    enable_few_shot: bool = True
+    num_shots: int = 5
+    
+    # Experiment tracking
+    use_wandb: bool = False
+    wandb_project: Optional[str] = None
+    wandb_entity: Optional[str] = None
+    use_mlflow: bool = False
+    mlflow_tracking_uri: Optional[str] = None
+    
+    # Results settings
+    save_predictions: bool = True
+    save_detailed_results: bool = True
+    export_format: str = "json"  # json, csv, html
+    
+    def __post_init__(self):
+        """Initialize default values after creation."""
+        if self.custom_metrics is None:
+            self.custom_metrics = []
+        
+        # Ensure output directory exists
+        os.makedirs(self.output_dir, exist_ok=True)
+    
+    @classmethod
+    def from_dict(cls, config_dict: Dict[str, Any]) -> 'EvaluationConfig':
+        """
+        Create configuration from dictionary.
+        
+        Args:
+            config_dict: Configuration dictionary
+            
+        Returns:
+            EvaluationConfig instance
+        """
+        # Filter out unknown keys
+        valid_keys = {field.name for field in cls.__dataclass_fields__.values()}
+        filtered_dict = {k: v for k, v in config_dict.items() if k in valid_keys}
+        
+        return cls(**filtered_dict)
+    
+    def to_dict(self) -> Dict[str, Any]:
+        """
+        Convert configuration to dictionary.
+        
+        Returns:
+            Configuration as dictionary
+        """
+        return asdict(self)
+
+
+class ConfigManager:
+    """Manager for handling multiple evaluation configurations."""
+    
+    def __init__(self, config_dir: str = "configs"):
+        """Initialize configuration manager."""
+        self.config_dir = config_dir
+        self.configs: Dict[str, EvaluationConfig] = {}
+        self.default_config = EvaluationConfig()
+        
+        # Ensure config directory exists
+        os.makedirs(config_dir, exist_ok=True)
+    
+    def get_config(self, config_name: Optional[str] = None) -> EvaluationConfig:
+        """Get configuration by name."""
+        if config_name is None:
+            return self.default_config
+        
+        if config_name in self.configs:
+            return self.configs[config_name]
+        
+        return self.default_config

isa_model/eval/evaluators/__init__.py

@@ -0,0 +1,18 @@
+"""
+Evaluators module for ISA Model Framework
+
+Provides specialized evaluators for different model types and evaluation tasks.
+"""
+
+from .base_evaluator import BaseEvaluator, EvaluationResult
+from .llm_evaluator import LLMEvaluator
+from .vision_evaluator import VisionEvaluator
+from .multimodal_evaluator import MultimodalEvaluator
+
+__all__ = [
+    "BaseEvaluator",
+    "EvaluationResult", 
+    "LLMEvaluator",
+    "VisionEvaluator",
+    "MultimodalEvaluator"
+]