isa-model 0.3.4__py3-none-any.whl → 0.3.5__py3-none-any.whl

isa_model/config/__init__.py

@@ -0,0 +1,9 @@
+"""
+Configuration Management Module
+
+Unified configuration system for all ISA Model components
+"""
+
+from .config_manager import ConfigManager, get_config, DeploymentConfig, ModelConfig
+
+__all__ = ["ConfigManager", "get_config", "DeploymentConfig", "ModelConfig"]

isa_model/config/config_manager.py

@@ -0,0 +1,213 @@
+"""
+Configuration Manager
+
+Central configuration management with environment support
+"""
+
+import os
+import yaml
+from typing import Dict, Any, Optional
+from pathlib import Path
+from dataclasses import dataclass
+import logging
+
+logger = logging.getLogger(__name__)
+
+@dataclass
+class ConfigSection:
+    """Base configuration section"""
+    def to_dict(self) -> Dict[str, Any]:
+        return self.__dict__
+
+@dataclass 
+class DeploymentConfig(ConfigSection):
+    """Deployment configuration"""
+    platform: str = "replicate"  # replicate, modal, aws, local
+    modal_app_name: str = "isa-ui-analysis"
+    modal_gpu_type: str = "A100-40GB"
+    modal_memory: int = 32768
+    modal_timeout: int = 1800
+    modal_keep_warm: int = 1
+    
+@dataclass
+class ModelConfig(ConfigSection):
+    """Model configuration"""
+    ui_detection_model: str = "microsoft/omniparser-v2"
+    ui_planning_model: str = "gpt-4o-mini"
+    fallback_detection: str = "yolov8n"
+    quantization: bool = False
+    batch_size: int = 1
+
+@dataclass
+class ServingConfig(ConfigSection):
+    """Serving configuration"""
+    host: str = "0.0.0.0"
+    port: int = 8000
+    workers: int = 1
+    reload: bool = False
+    log_level: str = "info"
+    cors_origins: list = None
+    
+    def __post_init__(self):
+        if self.cors_origins is None:
+            self.cors_origins = ["*"]
+
+@dataclass
+class APIConfig(ConfigSection):
+    """API configuration"""
+    rate_limit: int = 100  # requests per minute
+    max_file_size: int = 10 * 1024 * 1024  # 10MB
+    cache_ttl: int = 3600  # 1 hour
+    enable_auth: bool = False
+
+@dataclass
+class ISAConfig:
+    """Complete ISA configuration"""
+    environment: str
+    deployment: DeploymentConfig
+    models: ModelConfig  
+    serving: ServingConfig
+    api: APIConfig
+    
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "environment": self.environment,
+            "deployment": self.deployment.to_dict(),
+            "models": self.models.to_dict(),
+            "serving": self.serving.to_dict(),
+            "api": self.api.to_dict()
+        }
+
+class ConfigManager:
+    """Configuration manager with environment support"""
+    
+    _instance = None
+    _config = None
+    
+    def __new__(cls):
+        if cls._instance is None:
+            cls._instance = super(ConfigManager, cls).__new__(cls)
+        return cls._instance
+    
+    def __init__(self):
+        if self._config is None:
+            self._load_config()
+    
+    def _load_config(self):
+        """Load configuration from environment and files"""
+        env = os.getenv("ISA_ENV", "development")
+        
+        # Default configurations
+        default_config = {
+            "deployment": DeploymentConfig(),
+            "models": ModelConfig(), 
+            "serving": ServingConfig(),
+            "api": APIConfig()
+        }
+        
+        # Load environment-specific configuration
+        config_file = self._get_config_file(env)
+        if config_file and config_file.exists():
+            try:
+                with open(config_file, 'r') as f:
+                    file_config = yaml.safe_load(f)
+                    
+                # Merge configurations
+                self._config = self._merge_configs(default_config, file_config, env)
+                logger.info(f"Loaded configuration for environment: {env}")
+                
+            except Exception as e:
+                logger.warning(f"Failed to load config file {config_file}: {e}")
+                self._config = ISAConfig(environment=env, **default_config)
+        else:
+            logger.info(f"No config file found for {env}, using defaults")
+            self._config = ISAConfig(environment=env, **default_config)
+        
+        # Override with environment variables
+        self._apply_env_overrides()
+    
+    def _get_config_file(self, env: str) -> Optional[Path]:
+        """Get configuration file path for environment"""
+        # Try to find config file in multiple locations
+        possible_paths = [
+            Path(__file__).parent / "environments" / f"{env}.yaml",
+            Path.cwd() / "config" / f"{env}.yaml",
+            Path.cwd() / f"config_{env}.yaml"
+        ]
+        
+        for path in possible_paths:
+            if path.exists():
+                return path
+        return None
+    
+    def _merge_configs(self, default: Dict, file_config: Dict, env: str) -> ISAConfig:
+        """Merge default and file configurations"""
+        
+        # Update deployment config
+        deployment_data = {**default["deployment"].__dict__}
+        if "deployment" in file_config:
+            deployment_data.update(file_config["deployment"])
+        deployment = DeploymentConfig(**deployment_data)
+        
+        # Update model config
+        models_data = {**default["models"].__dict__}
+        if "models" in file_config:
+            models_data.update(file_config["models"])
+        models = ModelConfig(**models_data)
+        
+        # Update serving config
+        serving_data = {**default["serving"].__dict__}
+        if "serving" in file_config:
+            serving_data.update(file_config["serving"])
+        serving = ServingConfig(**serving_data)
+        
+        # Update API config
+        api_data = {**default["api"].__dict__}
+        if "api" in file_config:
+            api_data.update(file_config["api"])
+        api = APIConfig(**api_data)
+        
+        return ISAConfig(
+            environment=env,
+            deployment=deployment,
+            models=models,
+            serving=serving,
+            api=api
+        )
+    
+    def _apply_env_overrides(self):
+        """Apply environment variable overrides"""
+        # Deployment overrides
+        if os.getenv("ISA_DEPLOYMENT_PLATFORM"):
+            self._config.deployment.platform = os.getenv("ISA_DEPLOYMENT_PLATFORM")
+        
+        # Model overrides
+        if os.getenv("ISA_UI_DETECTION_MODEL"):
+            self._config.models.ui_detection_model = os.getenv("ISA_UI_DETECTION_MODEL")
+        
+        # Serving overrides
+        if os.getenv("ISA_SERVING_PORT"):
+            self._config.serving.port = int(os.getenv("ISA_SERVING_PORT"))
+        
+        if os.getenv("ISA_SERVING_HOST"):
+            self._config.serving.host = os.getenv("ISA_SERVING_HOST")
+    
+    def get_config(self) -> ISAConfig:
+        """Get current configuration"""
+        return self._config
+    
+    def reload(self):
+        """Reload configuration"""
+        self._config = None
+        self._load_config()
+
+# Singleton instance
+_config_manager = ConfigManager()
+
+def get_config() -> ISAConfig:
+    """Get configuration instance"""
+    return _config_manager.get_config()
+
+def reload_config():
+    """Reload configuration"""
+    _config_manager.reload()

isa_model/core/model_manager.py

@@ -41,6 +41,11 @@ class ModelManager:
             "meta/meta-llama-3-8b-instruct": {"input": 0.05, "output": 0.25},
             "kokoro-82m": {"input": 0.0, "output": 0.4},  # ~$0.0004 per second
             "jaaari/kokoro-82m:f559560eb822dc509045f3921a1921234918b91739db4bf3daab2169b71c7a13": {"input": 0.0, "output": 0.4},
+        },
+        # YYDS Models
+        "yyds": {
+            "claude-sonnet-4-20250514": {"input": 4.5, "output": 22.5},  # $0.0045/1K = $4.5/1M, $0.0225/1K = $22.5/1M
+            "claude-3-5-sonnet-20241022": {"input": 3.0, "output": 15.0},  # $0.003/1K = $3.0/1M, $0.015/1K = $15.0/1M
         }
     }
     

isa_model/core/model_registry.py

@@ -20,6 +20,10 @@ class ModelCapability(str, Enum):
     IMAGE_ANALYSIS = "image_analysis"
     AUDIO_TRANSCRIPTION = "audio_transcription"
     IMAGE_UNDERSTANDING = "image_understanding"
+    UI_DETECTION = "ui_detection"
+    OCR = "ocr"
+    TABLE_DETECTION = "table_detection"
+    TABLE_STRUCTURE_RECOGNITION = "table_structure_recognition"
 
 class ModelType(str, Enum):
     """Model types"""
@@ -32,13 +36,33 @@ class ModelType(str, Enum):
     VISION = "vision"
 
 class ModelRegistry:
-    """SQLite-based registry for model metadata and capabilities"""
+    """Model registry with SQLite or Supabase backend"""
     
-    def __init__(self, db_path: str = "./models/model_registry.db"):
-        self.db_path = Path(db_path)
-        self.db_path.parent.mkdir(parents=True, exist_ok=True)
-        self._lock = threading.Lock()
-        self._initialize_database()
+    def __init__(self, db_path: str = "./models/model_registry.db", use_supabase: bool = None):
+        # Auto-detect Supabase if environment variables are set
+        if use_supabase is None:
+            import os
+            use_supabase = bool(os.getenv("SUPABASE_URL") and os.getenv("SUPABASE_ANON_KEY"))
+        
+        self.use_supabase = use_supabase
+        
+        if self.use_supabase:
+            try:
+                from .storage.supabase_storage import SupabaseModelRegistry
+                self.backend = SupabaseModelRegistry()
+                logger.info("Using Supabase backend for model registry")
+            except ImportError as e:
+                logger.warning(f"Supabase not available, falling back to SQLite: {e}")
+                self.use_supabase = False
+        
+        if not self.use_supabase:
+            # Use SQLite backend
+            self.db_path = Path(db_path)
+            self.db_path.parent.mkdir(parents=True, exist_ok=True)
+            self._lock = threading.Lock()
+            self._initialize_database()
+            self.backend = None
+            logger.info("Using SQLite backend for model registry")
     
     def _initialize_database(self):
         """Initialize SQLite database with required tables"""
@@ -78,6 +102,15 @@ class ModelRegistry:
                       capabilities: List[ModelCapability],
                       metadata: Dict[str, Any]) -> bool:
         """Register a model with its capabilities and metadata"""
+        if self.use_supabase:
+            return self.backend.register_model(
+                model_id=model_id,
+                model_type=model_type.value,
+                capabilities=[cap.value for cap in capabilities],
+                metadata=metadata
+            )
+        
+        # SQLite implementation
         try:
             with self._lock:
                 with sqlite3.connect(self.db_path) as conn:

isa_model/core/storage/supabase_storage.py

@@ -0,0 +1,344 @@
+"""
+Supabase Storage Implementation for Model Registry
+
+Uses Supabase as the backend database for model metadata and capabilities
+Supports the full model lifecycle with cloud-based storage
+"""
+
+import os
+import json
+import logging
+from typing import Optional, Dict, Any, List
+from datetime import datetime
+from pathlib import Path
+
+try:
+    from supabase import create_client, Client
+    from dotenv import load_dotenv
+    SUPABASE_AVAILABLE = True
+except ImportError:
+    SUPABASE_AVAILABLE = False
+
+from ..model_storage import ModelStorage
+
+logger = logging.getLogger(__name__)
+
+class SupabaseModelRegistry:
+    """
+    Supabase-based model registry for metadata and capabilities
+    
+    Replaces SQLite with cloud-based Supabase database
+    """
+    
+    def __init__(self):
+        if not SUPABASE_AVAILABLE:
+            raise ImportError("supabase-py is required. Install with: pip install supabase")
+        
+        # Load environment variables
+        load_dotenv()
+        
+        self.supabase_url = os.getenv("SUPABASE_URL")
+        self.supabase_key = os.getenv("SUPABASE_ANON_KEY")
+        
+        if not self.supabase_url or not self.supabase_key:
+            raise ValueError("SUPABASE_URL and SUPABASE_ANON_KEY must be set in environment")
+        
+        # Initialize Supabase client
+        self.supabase: Client = create_client(self.supabase_url, self.supabase_key)
+        
+        # Initialize tables if needed
+        self._ensure_tables()
+        
+        logger.info("Supabase model registry initialized")
+    
+    def _ensure_tables(self):
+        """Ensure required tables exist in Supabase"""
+        # Note: In production, these tables should be created via Supabase migrations
+        # This is just for development/initialization
+        try:
+            # Check if models table exists by trying to query it
+            result = self.supabase.table('models').select('model_id').limit(1).execute()
+        except Exception as e:
+            logger.warning(f"Models table might not exist: {e}")
+            # In production, you would run proper migrations here
+    
+    def register_model(self, 
+                      model_id: str,
+                      model_type: str,
+                      capabilities: List[str],
+                      metadata: Dict[str, Any]) -> bool:
+        """Register a model with its capabilities and metadata"""
+        try:
+            current_time = datetime.now().isoformat()
+            
+            # Prepare model data
+            model_data = {
+                'model_id': model_id,
+                'model_type': model_type,
+                'metadata': json.dumps(metadata),
+                'created_at': current_time,
+                'updated_at': current_time
+            }
+            
+            # Insert or update model
+            result = self.supabase.table('models').upsert(model_data).execute()
+            
+            if not result.data:
+                logger.error(f"Failed to insert model {model_id}")
+                return False
+            
+            # Delete existing capabilities
+            self.supabase.table('model_capabilities').delete().eq('model_id', model_id).execute()
+            
+            # Insert new capabilities
+            if capabilities:
+                capability_data = [
+                    {
+                        'model_id': model_id,
+                        'capability': capability,
+                        'created_at': current_time
+                    }
+                    for capability in capabilities
+                ]
+                
+                cap_result = self.supabase.table('model_capabilities').insert(capability_data).execute()
+                
+                if not cap_result.data:
+                    logger.error(f"Failed to insert capabilities for {model_id}")
+                    return False
+            
+            logger.info(f"Successfully registered model {model_id}")
+            return True
+            
+        except Exception as e:
+            logger.error(f"Failed to register model {model_id}: {e}")
+            return False
+    
+    def unregister_model(self, model_id: str) -> bool:
+        """Unregister a model"""
+        try:
+            # Delete model (capabilities will be cascade deleted)
+            result = self.supabase.table('models').delete().eq('model_id', model_id).execute()
+            
+            if result.data:
+                logger.info(f"Unregistered model {model_id}")
+                return True
+            return False
+            
+        except Exception as e:
+            logger.error(f"Failed to unregister model {model_id}: {e}")
+            return False
+    
+    def get_model_info(self, model_id: str) -> Optional[Dict[str, Any]]:
+        """Get model information"""
+        try:
+            # Get model info
+            model_result = self.supabase.table('models').select('*').eq('model_id', model_id).execute()
+            
+            if not model_result.data:
+                return None
+            
+            model_row = model_result.data[0]
+            
+            # Get capabilities
+            cap_result = self.supabase.table('model_capabilities').select('capability').eq('model_id', model_id).execute()
+            capabilities = [cap['capability'] for cap in cap_result.data]
+            
+            model_info = {
+                "model_id": model_row["model_id"],
+                "type": model_row["model_type"],
+                "capabilities": capabilities,
+                "metadata": json.loads(model_row["metadata"]) if model_row["metadata"] else {},
+                "created_at": model_row["created_at"],
+                "updated_at": model_row["updated_at"]
+            }
+            
+            return model_info
+            
+        except Exception as e:
+            logger.error(f"Failed to get model info for {model_id}: {e}")
+            return None
+    
+    def get_models_by_type(self, model_type: str) -> Dict[str, Dict[str, Any]]:
+        """Get all models of a specific type"""
+        try:
+            models_result = self.supabase.table('models').select('*').eq('model_type', model_type).execute()
+            
+            result = {}
+            for model in models_result.data:
+                model_id = model["model_id"]
+                
+                # Get capabilities for this model
+                cap_result = self.supabase.table('model_capabilities').select('capability').eq('model_id', model_id).execute()
+                capabilities = [cap['capability'] for cap in cap_result.data]
+                
+                result[model_id] = {
+                    "type": model["model_type"],
+                    "capabilities": capabilities,
+                    "metadata": json.loads(model["metadata"]) if model["metadata"] else {},
+                    "created_at": model["created_at"],
+                    "updated_at": model["updated_at"]
+                }
+            
+            return result
+            
+        except Exception as e:
+            logger.error(f"Failed to get models by type {model_type}: {e}")
+            return {}
+    
+    def get_models_by_capability(self, capability: str) -> Dict[str, Dict[str, Any]]:
+        """Get all models with a specific capability"""
+        try:
+            # Join query to get models with specific capability
+            query = """
+            SELECT DISTINCT m.*, mc.capability
+            FROM models m
+            INNER JOIN model_capabilities mc ON m.model_id = mc.model_id
+            WHERE mc.capability = %s
+            """
+            
+            # Use RPC for complex queries
+            result = self.supabase.rpc('get_models_by_capability', {'capability_name': capability}).execute()
+            
+            if result.data:
+                models_dict = {}
+                for row in result.data:
+                    model_id = row['model_id']
+                    if model_id not in models_dict:
+                        # Get all capabilities for this model
+                        cap_result = self.supabase.table('model_capabilities').select('capability').eq('model_id', model_id).execute()
+                        capabilities = [cap['capability'] for cap in cap_result.data]
+                        
+                        models_dict[model_id] = {
+                            "type": row["model_type"],
+                            "capabilities": capabilities,
+                            "metadata": json.loads(row["metadata"]) if row["metadata"] else {},
+                            "created_at": row["created_at"],
+                            "updated_at": row["updated_at"]
+                        }
+                
+                return models_dict
+            
+            # Fallback: manual join if RPC not available
+            cap_result = self.supabase.table('model_capabilities').select('model_id').eq('capability', capability).execute()
+            model_ids = [row['model_id'] for row in cap_result.data]
+            
+            if not model_ids:
+                return {}
+            
+            models_result = self.supabase.table('models').select('*').in_('model_id', model_ids).execute()
+            
+            result_dict = {}
+            for model in models_result.data:
+                model_id = model["model_id"]
+                
+                # Get all capabilities for this model
+                all_caps_result = self.supabase.table('model_capabilities').select('capability').eq('model_id', model_id).execute()
+                capabilities = [cap['capability'] for cap in all_caps_result.data]
+                
+                result_dict[model_id] = {
+                    "type": model["model_type"],
+                    "capabilities": capabilities,
+                    "metadata": json.loads(model["metadata"]) if model["metadata"] else {},
+                    "created_at": model["created_at"],
+                    "updated_at": model["updated_at"]
+                }
+            
+            return result_dict
+            
+        except Exception as e:
+            logger.error(f"Failed to get models by capability {capability}: {e}")
+            return {}
+    
+    def has_capability(self, model_id: str, capability: str) -> bool:
+        """Check if a model has a specific capability"""
+        try:
+            result = self.supabase.table('model_capabilities').select('model_id').eq('model_id', model_id).eq('capability', capability).execute()
+            
+            return len(result.data) > 0
+            
+        except Exception as e:
+            logger.error(f"Failed to check capability for {model_id}: {e}")
+            return False
+    
+    def list_models(self) -> Dict[str, Dict[str, Any]]:
+        """List all registered models"""
+        try:
+            models_result = self.supabase.table('models').select('*').order('created_at', desc=True).execute()
+            
+            result = {}
+            for model in models_result.data:
+                model_id = model["model_id"]
+                
+                # Get capabilities for this model
+                cap_result = self.supabase.table('model_capabilities').select('capability').eq('model_id', model_id).execute()
+                capabilities = [cap['capability'] for cap in cap_result.data]
+                
+                result[model_id] = {
+                    "type": model["model_type"],
+                    "capabilities": capabilities,
+                    "metadata": json.loads(model["metadata"]) if model["metadata"] else {},
+                    "created_at": model["created_at"],
+                    "updated_at": model["updated_at"]
+                }
+            
+            return result
+            
+        except Exception as e:
+            logger.error(f"Failed to list models: {e}")
+            return {}
+    
+    def get_stats(self) -> Dict[str, Any]:
+        """Get registry statistics"""
+        try:
+            # Count total models
+            total_result = self.supabase.table('models').select('model_id', count='exact').execute()
+            total_models = total_result.count if total_result.count is not None else 0
+            
+            # Count by type
+            type_result = self.supabase.rpc('get_model_type_counts').execute()
+            type_counts = {row['model_type']: row['count'] for row in type_result.data} if type_result.data else {}
+            
+            # Count by capability
+            cap_result = self.supabase.rpc('get_capability_counts').execute()
+            capability_counts = {row['capability']: row['count'] for row in cap_result.data} if cap_result.data else {}
+            
+            return {
+                "total_models": total_models,
+                "models_by_type": type_counts,
+                "models_by_capability": capability_counts
+            }
+            
+        except Exception as e:
+            logger.error(f"Failed to get stats: {e}")
+            return {"total_models": 0, "models_by_type": {}, "models_by_capability": {}}
+    
+    def search_models(self, query: str) -> Dict[str, Dict[str, Any]]:
+        """Search models by name or metadata"""
+        try:
+            # Search in model_id and metadata
+            models_result = self.supabase.table('models').select('*').or_(
+                f'model_id.ilike.%{query}%,metadata.ilike.%{query}%'
+            ).order('created_at', desc=True).execute()
+            
+            result = {}
+            for model in models_result.data:
+                model_id = model["model_id"]
+                
+                # Get capabilities for this model
+                cap_result = self.supabase.table('model_capabilities').select('capability').eq('model_id', model_id).execute()
+                capabilities = [cap['capability'] for cap in cap_result.data]
+                
+                result[model_id] = {
+                    "type": model["model_type"],
+                    "capabilities": capabilities,
+                    "metadata": json.loads(model["metadata"]) if model["metadata"] else {},
+                    "created_at": model["created_at"],
+                    "updated_at": model["updated_at"]
+                }
+            
+            return result
+            
+        except Exception as e:
+            logger.error(f"Failed to search models with query '{query}': {e}")
+            return {}