cinchdb 0.1.14__py3-none-any.whl → 0.1.17__py3-none-any.whl

cinchdb/core/tenant_activation.py

@@ -0,0 +1,216 @@
+"""Tenant activation and caching.
+
+This module handles local caching of tenant databases with
+LRU eviction and TTL management. WAL streaming is handled
+at the api-server layer, not in the core library.
+"""
+
+import os
+import sqlite3
+import hashlib
+from pathlib import Path
+from typing import Optional, Dict, Any
+from datetime import datetime, timedelta
+
+from cinchdb.core.connection import DatabaseConnection
+
+
+class TenantCache:
+    """Manages local cache of activated tenant databases.
+    
+    This implements an LRU cache with TTL for tenant SQLite files.
+    Tenants are activated on-demand from WAL replay and cached locally.
+    """
+    
+    def __init__(self, cache_dir: Optional[Path] = None, max_size_gb: float = 10.0):
+        """Initialize tenant cache.
+        
+        Args:
+            cache_dir: Directory for cached tenant databases
+            max_size_gb: Maximum cache size in GB before eviction
+        """
+        self.cache_dir = cache_dir or Path("/var/cache/cinchdb/tenants")
+        self.cache_dir.mkdir(parents=True, exist_ok=True)
+        self.max_size_bytes = int(max_size_gb * 1024 * 1024 * 1024)
+        
+        # Track cached tenants: tenant_key -> (path, version, last_access)
+        self.cache_index: Dict[str, Dict[str, Any]] = {}
+    
+    def _get_cache_path(self, database: str, branch: str, tenant: str) -> Path:
+        """Get cache path for a tenant database.
+        
+        Uses sharding to avoid too many files in one directory.
+        """
+        # Create tenant key
+        tenant_key = f"{database}_{branch}_{tenant}"
+        
+        # Calculate shard (2-level sharding like production)
+        hash_val = hashlib.sha256(tenant_key.encode()).hexdigest()
+        shard1 = hash_val[:2]
+        shard2 = hash_val[2:4]
+        
+        # Build path
+        cache_path = self.cache_dir / database / branch / shard1 / shard2
+        cache_path.mkdir(parents=True, exist_ok=True)
+        
+        return cache_path / f"{tenant}.db"
+    
+    def get_tenant_connection(
+        self,
+        database: str,
+        branch: str,
+        tenant: str,
+        wal_capture=None
+    ) -> DatabaseConnection:
+        """Get a database connection for a tenant.
+        
+        If the tenant is not cached, activates it from WAL replay.
+        
+        Args:
+            database: Database name
+            branch: Branch name
+            tenant: Tenant name
+            wal_capture: Optional WAL capture for streaming
+            
+        Returns:
+            DatabaseConnection to the tenant database
+        """
+        tenant_key = f"{database}_{branch}_{tenant}"
+        cache_path = self._get_cache_path(database, branch, tenant)
+        
+        # Check if already cached
+        if cache_path.exists():
+            # Update last access time
+            if tenant_key in self.cache_index:
+                self.cache_index[tenant_key]["last_access"] = datetime.now()
+            
+            return DatabaseConnection(cache_path, wal_capture=wal_capture)
+        
+        # Not cached - create empty database
+        # WAL streaming/activation happens at api-server layer if enabled
+        return self._create_empty_database(cache_path, wal_capture)
+    
+    def _create_empty_database(
+        self,
+        cache_path: Path,
+        wal_capture=None
+    ) -> DatabaseConnection:
+        """Create an empty database (for lazy tenants or non-streaming mode).
+        
+        Args:
+            cache_path: Path where database should be created
+            wal_capture: Optional WAL capture
+            
+        Returns:
+            DatabaseConnection to the new database
+        """
+        # Create empty SQLite database
+        conn = sqlite3.connect(str(cache_path))
+        
+        # Set up WAL mode
+        conn.execute("PRAGMA journal_mode = WAL")
+        conn.execute("PRAGMA synchronous = NORMAL")
+        conn.commit()
+        conn.close()
+        
+        return DatabaseConnection(cache_path, wal_capture=wal_capture)
+    
+    
+    def _evict_if_needed(self):
+        """Evict least recently used tenants if cache is too large."""
+        # Calculate total cache size
+        total_size = sum(
+            info["size_bytes"] for info in self.cache_index.values()
+        )
+        
+        if total_size <= self.max_size_bytes:
+            return  # Cache is within limits
+        
+        # Sort by last access time (LRU)
+        sorted_tenants = sorted(
+            self.cache_index.items(),
+            key=lambda x: x[1]["last_access"]
+        )
+        
+        # Evict until under limit
+        for tenant_key, info in sorted_tenants:
+            if total_size <= self.max_size_bytes:
+                break
+            
+            # Delete cached database
+            cache_path = info["path"]
+            if cache_path.exists():
+                # Delete SQLite files (main, WAL, SHM)
+                for suffix in ["", "-wal", "-shm"]:
+                    file_path = Path(str(cache_path) + suffix)
+                    if file_path.exists():
+                        file_path.unlink()
+                
+                print(f"Evicted cached tenant: {tenant_key}")
+            
+            # Remove from index
+            total_size -= info["size_bytes"]
+            del self.cache_index[tenant_key]
+    
+    def invalidate_tenant(self, database: str, branch: str, tenant: str):
+        """Invalidate a cached tenant (force re-activation on next access).
+        
+        Args:
+            database: Database name
+            branch: Branch name
+            tenant: Tenant name
+        """
+        tenant_key = f"{database}_{branch}_{tenant}"
+        
+        if tenant_key in self.cache_index:
+            info = self.cache_index[tenant_key]
+            cache_path = info["path"]
+            
+            # Delete cached files
+            if cache_path.exists():
+                for suffix in ["", "-wal", "-shm"]:
+                    file_path = Path(str(cache_path) + suffix)
+                    if file_path.exists():
+                        file_path.unlink()
+            
+            # Remove from index
+            del self.cache_index[tenant_key]
+            print(f"Invalidated cached tenant: {tenant_key}")
+    
+    def get_cache_stats(self) -> Dict[str, Any]:
+        """Get cache statistics.
+        
+        Returns:
+            Dictionary with cache stats
+        """
+        total_size = sum(
+            info["size_bytes"] for info in self.cache_index.values()
+        )
+        
+        return {
+            "cached_tenants": len(self.cache_index),
+            "total_size_bytes": total_size,
+            "total_size_mb": round(total_size / (1024 * 1024), 2),
+            "max_size_gb": self.max_size_bytes / (1024 * 1024 * 1024),
+            "cache_dir": str(self.cache_dir)
+        }
+
+
+# Global cache instance (singleton)
+_tenant_cache: Optional[TenantCache] = None
+
+
+def get_tenant_cache() -> TenantCache:
+    """Get the global tenant cache instance.
+    
+    Returns:
+        TenantCache singleton
+    """
+    global _tenant_cache
+    
+    if _tenant_cache is None:
+        cache_dir = os.getenv("TENANT_CACHE_DIR", "/var/cache/cinchdb")
+        cache_size_gb = float(os.getenv("TENANT_CACHE_SIZE_GB", "10.0"))
+        _tenant_cache = TenantCache(Path(cache_dir), cache_size_gb)
+    
+    return _tenant_cache

cinchdb/infrastructure/metadata_connection_pool.py

@@ -3,7 +3,6 @@
 import threading
 from pathlib import Path
 from typing import Optional, Dict
-from weakref import WeakValueDictionary
 
 from cinchdb.infrastructure.metadata_db import MetadataDB
 

cinchdb/infrastructure/metadata_db.py

@@ -4,7 +4,6 @@ import sqlite3
 import uuid
 from pathlib import Path
 from typing import Optional, List, Dict, Any
-from datetime import datetime
 import json
 
 
@@ -38,6 +37,7 @@ class MetadataDB:
         self.conn.execute("PRAGMA foreign_keys = ON")
         self.conn.execute("PRAGMA journal_mode = WAL")  # Better concurrency
     
+    
     def _create_tables(self):
         """Create the metadata tables if they don't exist."""
         with self.conn:
@@ -48,6 +48,9 @@ class MetadataDB:
                     name TEXT NOT NULL UNIQUE,
                     description TEXT,
                     materialized BOOLEAN DEFAULT FALSE,
+                    maintenance_mode BOOLEAN DEFAULT FALSE,
+                    maintenance_reason TEXT,
+                    maintenance_started_at TIMESTAMP,
                     metadata JSON,
                     created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                     updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
@@ -63,6 +66,9 @@ class MetadataDB:
                     parent_branch TEXT,
                     schema_version TEXT,
                     materialized BOOLEAN DEFAULT FALSE,
+                    maintenance_mode BOOLEAN DEFAULT FALSE,
+                    maintenance_reason TEXT,
+                    maintenance_started_at TIMESTAMP,
                     metadata JSON,
                     created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                     updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
@@ -117,6 +123,19 @@ class MetadataDB:
                 CREATE INDEX IF NOT EXISTS idx_tenants_shard 
                 ON tenants(shard)
             """)
+
+            try:
+                self.conn.execute("""
+                    ALTER TABLE branches ADD COLUMN cdc_enabled BOOLEAN DEFAULT FALSE
+                """)
+            except sqlite3.OperationalError:
+                # Column already exists
+                pass
+            
+            self.conn.execute("""
+                CREATE INDEX IF NOT EXISTS idx_branches_cdc_enabled 
+                ON branches(cdc_enabled)
+            """)
     
     # Database operations
     def create_database(self, database_id: str, name: str, 
@@ -362,6 +381,94 @@ class MetadataDB:
             
             return len(tenants)
     
+    # Maintenance Mode Methods
+    
+    def set_database_maintenance(self, database_name: str, enabled: bool, reason: Optional[str] = None) -> None:
+        """Set maintenance mode for a database."""
+        with self.conn:
+            if enabled:
+                self.conn.execute("""
+                    UPDATE databases 
+                    SET maintenance_mode = TRUE, 
+                        maintenance_reason = ?, 
+                        maintenance_started_at = CURRENT_TIMESTAMP 
+                    WHERE name = ?
+                """, (reason, database_name))
+            else:
+                self.conn.execute("""
+                    UPDATE databases 
+                    SET maintenance_mode = FALSE, 
+                        maintenance_reason = NULL, 
+                        maintenance_started_at = NULL 
+                    WHERE name = ?
+                """, (database_name,))
+    
+    def set_branch_maintenance(self, database_name: str, branch_name: str, enabled: bool, reason: Optional[str] = None) -> None:
+        """Set maintenance mode for a branch."""
+        with self.conn:
+            if enabled:
+                self.conn.execute("""
+                    UPDATE branches 
+                    SET maintenance_mode = TRUE, 
+                        maintenance_reason = ?, 
+                        maintenance_started_at = CURRENT_TIMESTAMP 
+                    WHERE database_id = (SELECT id FROM databases WHERE name = ?) 
+                    AND name = ?
+                """, (reason, database_name, branch_name))
+            else:
+                self.conn.execute("""
+                    UPDATE branches 
+                    SET maintenance_mode = FALSE, 
+                        maintenance_reason = NULL, 
+                        maintenance_started_at = NULL 
+                    WHERE database_id = (SELECT id FROM databases WHERE name = ?) 
+                    AND name = ?
+                """, (database_name, branch_name))
+    
+    def is_database_in_maintenance(self, database_name: str) -> bool:
+        """Check if database is in maintenance mode."""
+        cursor = self.conn.execute("""
+            SELECT maintenance_mode FROM databases WHERE name = ?
+        """, (database_name,))
+        row = cursor.fetchone()
+        return bool(row and row['maintenance_mode'])
+    
+    def is_branch_in_maintenance(self, database_name: str, branch_name: str) -> bool:
+        """Check if branch is in maintenance mode."""
+        cursor = self.conn.execute("""
+            SELECT maintenance_mode FROM branches 
+            WHERE database_id = (SELECT id FROM databases WHERE name = ?) 
+            AND name = ?
+        """, (database_name, branch_name))
+        row = cursor.fetchone()
+        return bool(row and row['maintenance_mode'])
+    
+    def get_maintenance_info(self, database_name: str, branch_name: Optional[str] = None) -> Optional[Dict[str, Any]]:
+        """Get maintenance mode information."""
+        if branch_name:
+            # Check branch maintenance
+            cursor = self.conn.execute("""
+                SELECT maintenance_mode, maintenance_reason, maintenance_started_at 
+                FROM branches 
+                WHERE database_id = (SELECT id FROM databases WHERE name = ?) 
+                AND name = ?
+            """, (database_name, branch_name))
+        else:
+            # Check database maintenance
+            cursor = self.conn.execute("""
+                SELECT maintenance_mode, maintenance_reason, maintenance_started_at 
+                FROM databases WHERE name = ?
+            """, (database_name,))
+        
+        row = cursor.fetchone()
+        if row and row['maintenance_mode']:
+            return {
+                'enabled': True,
+                'reason': row['maintenance_reason'],
+                'started_at': row['maintenance_started_at']
+            }
+        return None
+    
     def close(self):
         """Close the database connection."""
         if self.conn:

cinchdb/managers/branch.py

@@ -4,7 +4,7 @@ import json
 import shutil
 import uuid
 from pathlib import Path
-from typing import List, Dict, Any, Optional
+from typing import List, Dict, Any
 from datetime import datetime, timezone
 
 from cinchdb.models import Branch

cinchdb/managers/change_applier.py

@@ -11,6 +11,7 @@ from cinchdb.managers.change_tracker import ChangeTracker
 from cinchdb.managers.tenant import TenantManager
 from cinchdb.core.connection import DatabaseConnection
 from cinchdb.core.path_utils import get_tenant_db_path, get_branch_path
+from cinchdb.infrastructure.metadata_connection_pool import get_metadata_db
 
 logger = logging.getLogger(__name__)
 
@@ -388,21 +389,17 @@ class ChangeApplier:
 
     def _enter_maintenance_mode(self) -> None:
         """Enter maintenance mode to block writes during schema changes."""
-        # Create a maintenance mode file that all connections can check
-        branch_path = get_branch_path(self.project_root, self.database, self.branch)
-        maintenance_file = branch_path / ".maintenance_mode"
-
-        with open(maintenance_file, "w") as f:
-            import json
-
-            json.dump(
-                {
-                    "active": True,
-                    "reason": "Schema update in progress",
-                    "started_at": datetime.now().isoformat(),
-                },
-                f,
+        try:
+            metadata_db = get_metadata_db(self.project_root)
+            metadata_db.set_branch_maintenance(
+                self.database, 
+                self.branch, 
+                True, 
+                "Schema update in progress"
             )
+        except Exception as e:
+            logger.error(f"Failed to enter maintenance mode: {e}")
+            # Continue anyway - we'll try to proceed with the schema update
 
         # Give time for any in-flight writes to complete
         # Can be disabled for tests via environment variable
@@ -414,11 +411,11 @@ class ChangeApplier:
 
     def _exit_maintenance_mode(self) -> None:
         """Exit maintenance mode to allow writes again."""
-        branch_path = get_branch_path(self.project_root, self.database, self.branch)
-        maintenance_file = branch_path / ".maintenance_mode"
-
-        if maintenance_file.exists():
-            maintenance_file.unlink()
+        try:
+            metadata_db = get_metadata_db(self.project_root)
+            metadata_db.set_branch_maintenance(self.database, self.branch, False)
+        except Exception as e:
+            logger.error(f"Failed to exit maintenance mode: {e}")
 
     def is_in_maintenance_mode(self) -> bool:
         """Check if branch is in maintenance mode.
@@ -426,6 +423,8 @@ class ChangeApplier:
         Returns:
             True if in maintenance mode, False otherwise
         """
-        branch_path = get_branch_path(self.project_root, self.database, self.branch)
-        maintenance_file = branch_path / ".maintenance_mode"
-        return maintenance_file.exists()
+        try:
+            metadata_db = get_metadata_db(self.project_root)
+            return metadata_db.is_branch_in_maintenance(self.database, self.branch)
+        except Exception:
+            return False

cinchdb/managers/column.py

@@ -6,7 +6,7 @@ from typing import List, Any, Optional
 from cinchdb.models import Column, Change, ChangeType
 from cinchdb.core.connection import DatabaseConnection
 from cinchdb.core.path_utils import get_tenant_db_path
-from cinchdb.core.maintenance import check_maintenance_mode
+from cinchdb.core.maintenance_utils import check_maintenance_mode
 from cinchdb.managers.change_tracker import ChangeTracker
 from cinchdb.managers.table import TableManager