isa-model 0.3.91__py3-none-any.whl → 0.4.0__py3-none-any.whl

isa_model/core/database/migrations.py

@@ -0,0 +1,277 @@
+"""
+Database Migration Manager for ISA Model
+
+Handles schema creation and updates across different environments.
+Best practices:
+- Environment-specific migrations
+- Version-controlled schema changes
+- Rollback capabilities
+- Schema validation
+"""
+
+import logging
+import psycopg2
+from typing import Dict, List, Optional
+from pathlib import Path
+
+from ..config.config_manager import ConfigManager
+
+logger = logging.getLogger(__name__)
+
+class DatabaseMigrations:
+    """Manages database migrations across environments"""
+    
+    def __init__(self):
+        self.config_manager = ConfigManager()
+        self.global_config = self.config_manager.get_global_config()
+        self.environment = self.global_config.environment.value
+        self.schema = self.global_config.database.supabase_schema or "public"
+        
+    def get_database_url(self) -> str:
+        """Get the PostgreSQL connection URL"""
+        import os
+        return os.getenv('DATABASE_URL')
+    
+    def create_schema(self, schema_name: str) -> bool:
+        """Create a schema if it doesn't exist"""
+        try:
+            conn = psycopg2.connect(self.get_database_url())
+            cursor = conn.cursor()
+            
+            cursor.execute(f"CREATE SCHEMA IF NOT EXISTS {schema_name}")
+            conn.commit()
+            conn.close()
+            
+            logger.info(f"Schema '{schema_name}' created/verified")
+            return True
+            
+        except Exception as e:
+            logger.error(f"Failed to create schema '{schema_name}': {e}")
+            return False
+    
+    def get_schema_migrations(self) -> Dict[str, List[str]]:
+        """Define migrations for each schema"""
+        return {
+            "models_table": [
+                f"""
+                CREATE TABLE IF NOT EXISTS {self.schema}.models (
+                    model_id TEXT PRIMARY KEY,
+                    model_type TEXT NOT NULL,
+                    provider TEXT NOT NULL,
+                    metadata JSONB DEFAULT '{{}}',
+                    created_at TIMESTAMPTZ DEFAULT NOW(),
+                    updated_at TIMESTAMPTZ DEFAULT NOW()
+                );
+                """,
+                f"CREATE INDEX IF NOT EXISTS idx_{self.schema}_models_type ON {self.schema}.models(model_type);",
+                f"CREATE INDEX IF NOT EXISTS idx_{self.schema}_models_provider ON {self.schema}.models(provider);"
+            ],
+            "model_capabilities_table": [
+                f"""
+                CREATE TABLE IF NOT EXISTS {self.schema}.model_capabilities (
+                    model_id TEXT REFERENCES {self.schema}.models(model_id) ON DELETE CASCADE,
+                    capability TEXT NOT NULL,
+                    created_at TIMESTAMPTZ DEFAULT NOW(),
+                    PRIMARY KEY (model_id, capability)
+                );
+                """,
+                f"CREATE INDEX IF NOT EXISTS idx_{self.schema}_capabilities_capability ON {self.schema}.model_capabilities(capability);"
+            ],
+            # "model_usage_table": [
+            #     f"""
+            #     CREATE TABLE IF NOT EXISTS {self.schema}.model_usage (
+            #         id BIGSERIAL PRIMARY KEY,
+            #         timestamp TIMESTAMPTZ NOT NULL,
+            #         model_id TEXT NOT NULL,
+            #         operation_type TEXT NOT NULL,
+            #         provider TEXT NOT NULL,
+            #         service_type TEXT NOT NULL,
+            #         operation TEXT NOT NULL,
+            #         input_tokens INTEGER DEFAULT NULL,
+            #         output_tokens INTEGER DEFAULT NULL,
+            #         total_tokens INTEGER DEFAULT NULL,
+            #         input_units DECIMAL DEFAULT NULL,
+            #         output_units DECIMAL DEFAULT NULL,
+            #         cost_usd DECIMAL(12,8) DEFAULT 0,
+            #         metadata JSONB DEFAULT '{{}}',
+            #         created_at TIMESTAMPTZ DEFAULT NOW()
+            #     );
+            #     """,
+            #     f"CREATE INDEX IF NOT EXISTS idx_{self.schema}_usage_model_id ON {self.schema}.model_usage(model_id);",
+            #     f"CREATE INDEX IF NOT EXISTS idx_{self.schema}_usage_timestamp ON {self.schema}.model_usage(timestamp);",
+            #     f"CREATE INDEX IF NOT EXISTS idx_{self.schema}_usage_provider ON {self.schema}.model_usage(provider);",
+            #     f"CREATE INDEX IF NOT EXISTS idx_{self.schema}_usage_operation_type ON {self.schema}.model_usage(operation_type);"
+            # ],
+            "model_embeddings_table": [
+                # Try to create with VECTOR type, fallback to JSONB if pgvector not available
+                f"""
+                DO $$
+                BEGIN
+                    -- Try to create with VECTOR type
+                    BEGIN
+                        CREATE TABLE IF NOT EXISTS {self.schema}.model_embeddings (
+                            id BIGSERIAL PRIMARY KEY,
+                            model_id TEXT REFERENCES {self.schema}.models(model_id) ON DELETE CASCADE,
+                            provider TEXT NOT NULL,
+                            description TEXT NOT NULL,
+                            embedding VECTOR(1536),
+                            created_at TIMESTAMPTZ DEFAULT NOW(),
+                            updated_at TIMESTAMPTZ DEFAULT NOW()
+                        );
+                    EXCEPTION WHEN undefined_object THEN
+                        -- Fallback to JSONB if VECTOR type doesn't exist
+                        CREATE TABLE IF NOT EXISTS {self.schema}.model_embeddings (
+                            id BIGSERIAL PRIMARY KEY,
+                            model_id TEXT REFERENCES {self.schema}.models(model_id) ON DELETE CASCADE,
+                            provider TEXT NOT NULL,
+                            description TEXT NOT NULL,
+                            embedding JSONB,
+                            created_at TIMESTAMPTZ DEFAULT NOW(),
+                            updated_at TIMESTAMPTZ DEFAULT NOW()
+                        );
+                    END;
+                END $$;
+                """,
+                f"CREATE INDEX IF NOT EXISTS idx_{self.schema}_embeddings_model_id ON {self.schema}.model_embeddings(model_id);"
+            ],
+            "tool_embeddings_table": [
+                f"""
+                CREATE TABLE IF NOT EXISTS {self.schema}.tool_embeddings (
+                    id BIGSERIAL PRIMARY KEY,
+                    tool_name TEXT UNIQUE NOT NULL,
+                    description TEXT,
+                    embedding JSONB,
+                    created_at TIMESTAMPTZ DEFAULT NOW(),
+                    updated_at TIMESTAMPTZ DEFAULT NOW()
+                );
+                """,
+                f"CREATE INDEX IF NOT EXISTS idx_{self.schema}_tool_embeddings_name ON {self.schema}.tool_embeddings(tool_name);"
+            ],
+            "prompt_embeddings_table": [
+                f"""
+                CREATE TABLE IF NOT EXISTS {self.schema}.prompt_embeddings (
+                    id BIGSERIAL PRIMARY KEY,
+                    prompt_name TEXT UNIQUE NOT NULL,
+                    description TEXT,
+                    embedding JSONB,
+                    created_at TIMESTAMPTZ DEFAULT NOW(),
+                    updated_at TIMESTAMPTZ DEFAULT NOW()
+                );
+                """,
+                f"CREATE INDEX IF NOT EXISTS idx_{self.schema}_prompt_embeddings_name ON {self.schema}.prompt_embeddings(prompt_name);"
+            ],
+            "resource_embeddings_table": [
+                f"""
+                CREATE TABLE IF NOT EXISTS {self.schema}.resource_embeddings (
+                    id BIGSERIAL PRIMARY KEY,
+                    resource_uri TEXT UNIQUE NOT NULL,
+                    category TEXT,
+                    name TEXT,
+                    description TEXT,
+                    embedding JSONB,
+                    created_at TIMESTAMPTZ DEFAULT NOW(),
+                    updated_at TIMESTAMPTZ DEFAULT NOW()
+                );
+                """,
+                f"CREATE INDEX IF NOT EXISTS idx_{self.schema}_resource_embeddings_uri ON {self.schema}.resource_embeddings(resource_uri);",
+                f"CREATE INDEX IF NOT EXISTS idx_{self.schema}_resource_embeddings_category ON {self.schema}.resource_embeddings(category);"
+            ]
+        }
+    
+    def run_migrations(self) -> bool:
+        """Run all migrations for the current environment"""
+        try:
+            # First, ensure schema exists
+            if not self.create_schema(self.schema):
+                return False
+            
+            conn = psycopg2.connect(self.get_database_url())
+            cursor = conn.cursor()
+            
+            migrations = self.get_schema_migrations()
+            
+            for migration_name, sql_statements in migrations.items():
+                logger.info(f"Running migration: {migration_name}")
+                
+                for sql in sql_statements:
+                    try:
+                        cursor.execute(sql)
+                        logger.debug(f"Executed: {sql[:100]}...")
+                    except Exception as e:
+                        logger.error(f"Failed to execute SQL in {migration_name}: {e}")
+                        logger.error(f"SQL: {sql}")
+                        conn.rollback()
+                        return False
+                
+                conn.commit()
+                logger.info(f"Migration '{migration_name}' completed")
+            
+            conn.close()
+            logger.info(f"All migrations completed for {self.environment} environment (schema: {self.schema})")
+            return True
+            
+        except Exception as e:
+            logger.error(f"Migration failed: {e}")
+            return False
+    
+    def validate_schema(self) -> Dict[str, bool]:
+        """Validate that all required tables exist with correct structure"""
+        results = {}
+        
+        try:
+            conn = psycopg2.connect(self.get_database_url())
+            cursor = conn.cursor()
+            
+            # Check each required table
+            required_tables = ['models', 'model_capabilities', 'model_embeddings', 'tool_embeddings', 'prompt_embeddings', 'resource_embeddings']
+            
+            for table in required_tables:
+                cursor.execute(f"""
+                    SELECT COUNT(*) 
+                    FROM information_schema.tables 
+                    WHERE table_schema = '{self.schema}' AND table_name = '{table}'
+                """)
+                
+                exists = cursor.fetchone()[0] > 0
+                results[table] = exists
+                
+                if exists:
+                    logger.info(f"✅ Table {self.schema}.{table} exists")
+                else:
+                    logger.warning(f"❌ Table {self.schema}.{table} missing")
+            
+            conn.close()
+            
+        except Exception as e:
+            logger.error(f"Schema validation failed: {e}")
+            
+        return results
+
+def run_environment_migrations():
+    """Convenience function to run migrations for current environment"""
+    migrations = DatabaseMigrations()
+    
+    logger.info(f"Starting migrations for {migrations.environment} environment")
+    logger.info(f"Target schema: {migrations.schema}")
+    
+    # Run migrations
+    success = migrations.run_migrations()
+    
+    if success:
+        # Validate schema
+        validation = migrations.validate_schema()
+        all_valid = all(validation.values())
+        
+        if all_valid:
+            logger.info("🎯 All migrations completed and validated successfully!")
+        else:
+            logger.warning("⚠️ Migrations completed but validation found issues")
+            
+        return all_valid
+    else:
+        logger.error("❌ Migration failed")
+        return False
+
+if __name__ == "__main__":
+    # Run migrations when script is executed directly
+    run_environment_migrations()

isa_model/core/database/supabase_client.py

@@ -0,0 +1,123 @@
+"""
+Centralized Supabase Client for ISA Model Core
+
+Provides a singleton Supabase client instance that:
+- Gets configuration from ConfigManager
+- Handles environment-based schema selection
+- Provides a single point of database access for all services
+"""
+
+import logging
+from typing import Optional
+from supabase import create_client, Client
+
+from ..config.config_manager import ConfigManager
+
+logger = logging.getLogger(__name__)
+
+class SupabaseClient:
+    """Singleton Supabase client with environment-aware configuration"""
+    
+    _instance: Optional['SupabaseClient'] = None
+    _client: Optional[Client] = None
+    _initialized = False
+    
+    def __new__(cls):
+        if cls._instance is None:
+            cls._instance = super().__new__(cls)
+        return cls._instance
+    
+    def __init__(self):
+        if not self._initialized:
+            self._initialize_client()
+            SupabaseClient._initialized = True
+    
+    def _initialize_client(self):
+        """Initialize the Supabase client with configuration from ConfigManager"""
+        try:
+            # Get configuration from ConfigManager
+            config_manager = ConfigManager()
+            global_config = config_manager.get_global_config()
+            
+            # Get database configuration
+            self.url = global_config.database.supabase_url
+            self.key = global_config.database.supabase_key
+            self.schema = global_config.database.supabase_schema or "public"
+            self.environment = global_config.environment.value
+            
+            if not self.url or not self.key:
+                raise ValueError("Supabase URL and key must be configured")
+            
+            # Create the client
+            self._client = create_client(self.url, self.key)
+            
+            logger.info(f"Supabase client initialized for {self.environment} environment (schema: {self.schema})")
+            
+        except Exception as e:
+            logger.error(f"Failed to initialize Supabase client: {e}")
+            raise
+    
+    def get_client(self) -> Client:
+        """Get the Supabase client instance"""
+        if not self._client:
+            raise RuntimeError("Supabase client not initialized")
+        return self._client
+    
+    def table(self, table_name: str):
+        """Get a table with the correct schema"""
+        if not self._client:
+            raise RuntimeError("Supabase client not initialized")
+        
+        # Use the configured schema for the environment
+        if self.schema and self.schema != "public":
+            return self._client.schema(self.schema).table(table_name)
+        else:
+            return self._client.table(table_name)
+    
+    def rpc(self, function_name: str, params: Optional[dict] = None):
+        """Call an RPC function with the correct schema"""
+        if not self._client:
+            raise RuntimeError("Supabase client not initialized")
+        
+        # RPC functions typically use the public schema
+        # But we can extend this if needed for schema-specific functions
+        return self._client.rpc(function_name, params)
+    
+    def get_schema(self) -> str:
+        """Get the current schema being used"""
+        return self.schema
+    
+    def get_environment(self) -> str:
+        """Get the current environment"""
+        return self.environment
+    
+    def test_connection(self) -> bool:
+        """Test the database connection"""
+        try:
+            # Try a simple query to test connection
+            result = self.table('models').select('*').limit(1).execute()
+            logger.debug("Database connection test successful")
+            return True
+        except Exception as e:
+            logger.warning(f"Database connection test failed: {e}")
+            return False
+
+# Global singleton instance
+_supabase_client = None
+
+def get_supabase_client() -> SupabaseClient:
+    """Get the global Supabase client instance"""
+    global _supabase_client
+    if _supabase_client is None:
+        _supabase_client = SupabaseClient()
+    return _supabase_client
+
+def get_supabase_table(table_name: str):
+    """Convenience function to get a table with correct schema"""
+    client = get_supabase_client()
+    return client.table(table_name)
+
+def get_supabase_rpc(function_name: str, params: Optional[dict] = None):
+    """Convenience function to call RPC functions"""
+    client = get_supabase_client()
+    return client.rpc(function_name, params)

isa_model/core/models/__init__.py

@@ -0,0 +1,37 @@
+"""
+Core Models Module
+
+Provides model management, registry, and lifecycle components for the ISA Model SDK.
+
+This module includes:
+- ModelRegistry: Central model registry and metadata management
+- ModelManager: High-level model lifecycle management
+- ModelVersionManager: Version control and lineage tracking
+- ModelBillingTracker: Cost tracking and billing
+- ModelStatisticsTracker: Usage statistics and analytics
+"""
+
+from .model_repo import ModelRegistry, ModelType, ModelCapability
+from .model_manager import ModelManager
+from .model_version_manager import ModelVersionManager, ModelVersion, VersionType
+from .model_billing_tracker import ModelBillingTracker
+from .model_statistics_tracker import ModelStatisticsTracker
+
+__all__ = [
+    # Core registry and types
+    'ModelRegistry',
+    'ModelType', 
+    'ModelCapability',
+    
+    # Model management
+    'ModelManager',
+    
+    # Version management
+    'ModelVersionManager',
+    'ModelVersion',
+    'VersionType',
+    
+    # Tracking and analytics
+    'ModelBillingTracker',
+    'ModelStatisticsTracker'
+]

isa_model/core/models/model_billing_tracker.py

@@ -40,6 +40,7 @@ class ServiceType(Enum):
     IMAGE_GENERATION = "image_generation"
     AUDIO_STT = "audio_stt"
     AUDIO_TTS = "audio_tts"
+    AUDIO_REALTIME = "audio_realtime"
 
 @dataclass
 class ModelUsageRecord:
@@ -108,7 +109,7 @@ class ModelBillingTracker:
     def _load_data(self):
         """Load existing billing data from registry or local storage"""
         try:
-            if self.model_registry and hasattr(self.model_registry, 'supabase'):
+            if self.model_registry and hasattr(self.model_registry, 'supabase_client'):
                 # Load from Supabase
                 self._load_from_supabase()
             else:
@@ -119,33 +120,10 @@ class ModelBillingTracker:
             self.usage_records = []
     
     def _load_from_supabase(self):
-        """Load billing data from Supabase"""
-        try:
-            if not self.model_registry or not hasattr(self.model_registry, 'supabase'):
-                logger.warning("No Supabase client available for billing data loading")
-                self.usage_records = []
-                return
-            
-            # Query model_usage table for recent usage records (last 30 days)
-            from datetime import datetime, timedelta
-            thirty_days_ago = (datetime.now() - timedelta(days=30)).isoformat()
-            
-            result = self.model_registry.supabase.table('model_usage').select('*').gte('timestamp', thirty_days_ago).order('timestamp', desc=True).execute()
-            
-            if result.data:
-                self.usage_records = [
-                    ModelUsageRecord.from_dict(record) 
-                    for record in result.data
-                ]
-                logger.info(f"Loaded {len(self.usage_records)} billing records from Supabase")
-            else:
-                self.usage_records = []
-                logger.info("No billing records found in Supabase")
-                
-        except Exception as e:
-            logger.error(f"Failed to load billing data from Supabase: {e}")
-            # Fallback to empty records
-            self.usage_records = []
+        """Load billing data from Supabase - disabled as model_usage table removed"""
+        # Model usage table has been removed, so no data to load
+        self.usage_records = []
+        logger.info("Model usage tracking disabled - no billing records to load from Supabase")
     
     def _load_from_local(self):
         """Load billing data from local JSON file"""
@@ -161,7 +139,7 @@ class ModelBillingTracker:
     def _save_data(self):
         """Save billing data to registry or local storage"""
         try:
-            if self.model_registry and hasattr(self.model_registry, 'supabase'):
+            if self.model_registry and hasattr(self.model_registry, 'supabase_client'):
                 self._save_to_supabase()
             else:
                 self._save_to_local()
@@ -170,52 +148,55 @@ class ModelBillingTracker:
     
     def _save_to_supabase(self):
         """Save billing data to Supabase"""
-        try:
-            if not self.model_registry or not hasattr(self.model_registry, 'supabase'):
-                logger.warning("No Supabase client available for billing data saving")
-                return
-            
-            if not self.usage_records:
-                logger.debug("No usage records to save")
-                return
-            
-            # Convert usage records to dict format for Supabase
-            records_to_save = []
-            for record in self.usage_records:
-                record_dict = record.to_dict()
-                # Ensure all required fields are present and properly formatted
-                record_dict['created_at'] = record_dict.get('timestamp')
-                records_to_save.append(record_dict)
-            
-            # Insert records into model_usage table (upsert to handle duplicates)
-            result = self.model_registry.supabase.table('model_usage').upsert(
-                records_to_save,
-                on_conflict='timestamp,model_id,operation'  # Avoid duplicates based on these fields
-            ).execute()
-            
-            if result.data:
-                logger.info(f"Successfully saved {len(result.data)} billing records to Supabase")
-            else:
-                logger.warning("No records were saved to Supabase")
-                
-        except Exception as e:
-            logger.error(f"Failed to save billing data to Supabase: {e}")
-            # Fallback to local storage on Supabase failure
-            logger.info("Falling back to local storage for billing data")
-            self._save_to_local()
+        # Disabled to prevent model_usage table writes
+        # try:
+        #     if not self.model_registry or not hasattr(self.model_registry, 'supabase_client'):
+        #         logger.warning("No Supabase client available for billing data saving")
+        #         return
+        #     
+        #     if not self.usage_records:
+        #         logger.debug("No usage records to save")
+        #         return
+        #     
+        #     # Convert usage records to dict format for Supabase
+        #     records_to_save = []
+        #     for record in self.usage_records:
+        #         record_dict = record.to_dict()
+        #         # Ensure all required fields are present and properly formatted
+        #         record_dict['created_at'] = record_dict.get('timestamp')
+        #         records_to_save.append(record_dict)
+        #     
+        #     # Insert records into model_usage table (simple insert, let DB handle duplicates)
+        #     result = self.model_registry.supabase_client.table('model_usage').insert(
+        #         records_to_save
+        #     ).execute()
+        #     
+        #     if result.data:
+        #         logger.info(f"Successfully saved {len(result.data)} billing records to Supabase")
+        #     else:
+        #         logger.warning("No records were saved to Supabase")
+        #         
+        # except Exception as e:
+        #     logger.error(f"Failed to save billing data to Supabase: {e}")
+        #     # Fallback to local storage on Supabase failure
+        #     logger.info("Falling back to local storage for billing data")
+        #     self._save_to_local()
+        pass
     
     def _save_to_local(self):
         """Save billing data to local JSON file"""
-        self.storage_path.parent.mkdir(parents=True, exist_ok=True)
-        
-        data = {
-            "session_start": self.session_start,
-            "last_updated": datetime.now(timezone.utc).isoformat(),
-            "usage_records": [record.to_dict() for record in self.usage_records]
-        }
-        
-        with open(self.storage_path, 'w') as f:
-            json.dump(data, f, indent=2)
+        # Disabled to prevent automatic file creation
+        # self.storage_path.parent.mkdir(parents=True, exist_ok=True)
+        # 
+        # data = {
+        #     "session_start": self.session_start,
+        #     "last_updated": datetime.now(timezone.utc).isoformat(),
+        #     "usage_records": [record.to_dict() for record in self.usage_records]
+        # }
+        # 
+        # with open(self.storage_path, 'w') as f:
+        #     json.dump(data, f, indent=2)
+        pass
     
     def track_model_usage(
         self,
@@ -307,23 +288,14 @@ class ModelBillingTracker:
             # Import here to avoid circular imports
             from .model_manager import ModelManager
             
-            # Get model info to determine provider model name
-            if self.model_registry:
-                model_info = self.model_registry.get_model_info(model_id)
-                if model_info and model_info.get('metadata'):
-                    provider_model_name = model_info['metadata'].get('provider_model_name')
-                    if provider_model_name:
-                        # Use ModelManager pricing
-                        pricing = ModelManager.MODEL_PRICING.get(provider, {}).get(provider_model_name)
-                        if pricing:
-                            cost = 0.0
-                            if input_tokens is not None and "input" in pricing:
-                                cost += (input_tokens / 1000000) * pricing["input"]
-                            if output_tokens is not None and "output" in pricing:
-                                cost += (output_tokens / 1000000) * pricing["output"]
-                            return cost
+            # Create ModelManager instance to get pricing
+            model_manager = ModelManager()
+            
+            # Use the centralized pricing calculation
+            if input_tokens is not None and output_tokens is not None:
+                return model_manager.calculate_cost(provider, model_id, input_tokens, output_tokens)
             
-            # Fallback to default pricing if model not found
+            # Fallback for non-token based services
             return 0.0
             
         except Exception as e: