cinchdb 0.1.15__py3-none-any.whl → 0.1.18__py3-none-any.whl

cinchdb/core/path_utils.py

@@ -1,7 +1,15 @@
 """Path utilities for CinchDB."""
 
+import hashlib
 from pathlib import Path
-from typing import List
+from typing import List, Dict, Tuple, Optional
+from cinchdb.infrastructure.metadata_connection_pool import get_metadata_db
+from cinchdb.utils.name_validator import validate_name
+
+# Cache for path calculations to avoid repeated string operations
+_path_cache: Dict[Tuple[str, str, str], Path] = {}
+_shard_cache: Dict[str, str] = {}
+_MAX_CACHE_SIZE = 10000  # Limit cache size to prevent unbounded growth
 
 
 def get_project_root(start_path: Path) -> Path:
@@ -40,7 +48,7 @@ def get_database_path(project_root: Path, database: str) -> Path:
 
 
 def get_branch_path(project_root: Path, database: str, branch: str) -> Path:
-    """Get path to a branch directory.
+    """Get path to a branch directory (now uses context root).
 
     Args:
         project_root: Project root directory
@@ -48,15 +56,16 @@ def get_branch_path(project_root: Path, database: str, branch: str) -> Path:
         branch: Branch name
 
     Returns:
-        Path to branch directory
+        Path to branch directory (context root in tenant-first structure)
     """
-    return get_database_path(project_root, database) / "branches" / branch
+    # In tenant-first structure, branch path is the context root
+    return get_context_root(project_root, database, branch)
 
 
 def get_tenant_path(
     project_root: Path, database: str, branch: str, tenant: str
 ) -> Path:
-    """Get path to tenant directory.
+    """Get path to tenant directory (deprecated, use get_context_root instead).
 
     Args:
         project_root: Project root directory
@@ -65,9 +74,9 @@ def get_tenant_path(
         tenant: Tenant name
 
     Returns:
-        Path to tenant directory
+        Path to context root (no longer has /tenants subfolder)
     """
-    return get_branch_path(project_root, database, branch) / "tenants"
+    return get_branch_path(project_root, database, branch)
 
 
 def get_tenant_db_path(
@@ -83,18 +92,20 @@ def get_tenant_db_path(
 
     Returns:
         Path to tenant database file in sharded directory structure
+        
+    Raises:
+        InvalidNameError: If any name contains path traversal characters
     """
-    import hashlib
-    
-    # Calculate shard using SHA256 hash (same as TenantManager)
-    hash_val = hashlib.sha256(tenant.encode('utf-8')).hexdigest()
-    shard = hash_val[:2]
+    # Validate all names to prevent path traversal
+    # Exception: __empty__ is a system tenant and exempt from validation
+    validate_name(database, "database")
+    validate_name(branch, "branch")
+    if tenant != "__empty__":  # System tenant exempt from validation
+        validate_name(tenant, "tenant")
     
-    # Build sharded path: /tenants/{shard}/{tenant}.db
-    tenants_dir = get_tenant_path(project_root, database, branch, tenant)
-    shard_dir = tenants_dir / shard
-    
-    return shard_dir / f"{tenant}.db"
+    context_root = get_context_root(project_root, database, branch)
+    shard = calculate_shard(tenant)
+    return context_root / shard / f"{tenant}.db"
 
 
 def ensure_directory(path: Path) -> None:
@@ -119,10 +130,9 @@ def list_databases(project_root: Path) -> List[str]:
     if not metadata_db_path.exists():
         return []
     
-    from cinchdb.infrastructure.metadata_db import MetadataDB
-    with MetadataDB(project_root) as metadata_db:
-        db_records = metadata_db.list_databases()
-        return sorted(record['name'] for record in db_records)
+    metadata_db = get_metadata_db(project_root)
+    db_records = metadata_db.list_databases()
+    return sorted(record['name'] for record in db_records)
 
 
 def list_branches(project_root: Path, database: str) -> List[str]:
@@ -139,13 +149,12 @@ def list_branches(project_root: Path, database: str) -> List[str]:
     if not metadata_db_path.exists():
         return []
     
-    from cinchdb.infrastructure.metadata_db import MetadataDB
-    with MetadataDB(project_root) as metadata_db:
-        db_info = metadata_db.get_database(database)
-        if not db_info:
-            return []
-        branch_records = metadata_db.list_branches(db_info['id'])
-        return sorted(record['name'] for record in branch_records)
+    metadata_db = get_metadata_db(project_root)
+    db_info = metadata_db.get_database(database)
+    if not db_info:
+        return []
+    branch_records = metadata_db.list_branches(db_info['id'])
+    return sorted(record['name'] for record in branch_records)
 
 
 def list_tenants(project_root: Path, database: str, branch: str) -> List[str]:
@@ -163,13 +172,177 @@ def list_tenants(project_root: Path, database: str, branch: str) -> List[str]:
     if not metadata_db_path.exists():
         return []
     
-    from cinchdb.infrastructure.metadata_db import MetadataDB
-    with MetadataDB(project_root) as metadata_db:
-        db_info = metadata_db.get_database(database)
-        if not db_info:
-            return []
-        branch_info = metadata_db.get_branch(db_info['id'], branch)
-        if not branch_info:
-            return []
-        tenant_records = metadata_db.list_tenants(branch_info['id'])
-        return sorted(record['name'] for record in tenant_records)
+    metadata_db = get_metadata_db(project_root)
+    db_info = metadata_db.get_database(database)
+    if not db_info:
+        return []
+    branch_info = metadata_db.get_branch(db_info['id'], branch)
+    if not branch_info:
+        return []
+    tenant_records = metadata_db.list_tenants(branch_info['id'])
+    return sorted(record['name'] for record in tenant_records)
+
+
+# New tenant-first path utilities
+
+def get_context_root(project_root: Path, database: str, branch: str) -> Path:
+    """Get root directory for a database-branch context with caching.
+    
+    This is the new tenant-first approach where each database-branch combination
+    gets its own isolated directory with tenants stored as a flat hierarchy.
+    
+    Args:
+        project_root: Project root directory
+        database: Database name
+        branch: Branch name
+        
+    Returns:
+        Path to context root directory (e.g., .cinchdb/prod-main/)
+    """
+    cache_key = (str(project_root), database, branch)
+    
+    if cache_key not in _path_cache:
+        # Check cache size and clear if needed
+        if len(_path_cache) >= _MAX_CACHE_SIZE:
+            _path_cache.clear()
+        
+        _path_cache[cache_key] = project_root / ".cinchdb" / f"{database}-{branch}"
+    
+    return _path_cache[cache_key]
+
+
+def calculate_shard(tenant_name: str) -> str:
+    """Calculate the shard directory for a tenant using SHA256 hash with caching.
+    
+    Args:
+        tenant_name: Name of the tenant
+        
+    Returns:
+        Two-character hex string (e.g., "a0", "ff")
+    """
+    if tenant_name not in _shard_cache:
+        # Check cache size and clear if needed
+        if len(_shard_cache) >= _MAX_CACHE_SIZE:
+            _shard_cache.clear()
+        
+        hash_val = hashlib.sha256(tenant_name.encode('utf-8')).hexdigest()
+        _shard_cache[tenant_name] = hash_val[:2]
+    
+    return _shard_cache[tenant_name]
+
+
+def get_tenant_db_path_in_context(context_root: Path, tenant: str) -> Path:
+    """Get tenant DB path within a context root.
+    
+    Uses the tenant-first storage approach where tenants are stored as:
+    {context_root}/{shard}/{tenant}.db
+    
+    Args:
+        context_root: Context root directory (from get_context_root)
+        tenant: Tenant name
+        
+    Returns:
+        Path to tenant database file within the context
+    """
+    shard = calculate_shard(tenant)
+    return context_root / shard / f"{tenant}.db"
+
+
+
+
+def ensure_context_directory(project_root: Path, database: str, branch: str) -> Path:
+    """Ensure context root directory exists and return it.
+    
+    Args:
+        project_root: Project root directory
+        database: Database name
+        branch: Branch name
+        
+    Returns:
+        Path to context root directory (created if necessary)
+    """
+    context_root = get_context_root(project_root, database, branch)
+    context_root.mkdir(parents=True, exist_ok=True)
+    
+    return context_root
+
+
+def ensure_tenant_db_path(project_root: Path, database: str, branch: str, tenant: str) -> Path:
+    """Ensure tenant database path exists (creates shard directory if needed).
+    
+    This is the ONLY place where tenant shard directories should be created,
+    ensuring we have a single source of truth for the directory structure.
+    
+    Args:
+        project_root: Project root directory
+        database: Database name
+        branch: Branch name
+        tenant: Tenant name
+        
+    Returns:
+        Path to tenant database file (directory created if necessary)
+        
+    Raises:
+        InvalidNameError: If any name contains path traversal characters
+    """
+    # Validation happens inside get_tenant_db_path
+    db_path = get_tenant_db_path(project_root, database, branch, tenant)
+    
+    # Ensure the shard directory exists
+    db_path.parent.mkdir(parents=True, exist_ok=True)
+    
+    return db_path
+
+
+def invalidate_cache(database: Optional[str] = None, 
+                    branch: Optional[str] = None, 
+                    tenant: Optional[str] = None) -> None:
+    """Invalidate cache entries (write-through on delete operations).
+    
+    This function is called when databases, branches, or tenants are deleted
+    to ensure cached paths don't point to non-existent resources.
+    
+    Args:
+        database: Database name to invalidate (invalidates all branches/tenants if provided)
+        branch: Branch name to invalidate (requires database)
+        tenant: Tenant name to invalidate (only invalidates shard cache)
+    """
+    global _path_cache, _shard_cache
+    
+    if database:
+        # Remove all cache entries for this database
+        keys_to_remove = []
+        for key in _path_cache:
+            # key is (project_root, database, branch)
+            if key[1] == database:
+                if branch is None or key[2] == branch:
+                    keys_to_remove.append(key)
+        
+        for key in keys_to_remove:
+            del _path_cache[key]
+    
+    if tenant and tenant in _shard_cache:
+        del _shard_cache[tenant]
+
+
+def clear_all_caches() -> None:
+    """Clear all path and shard caches.
+    
+    Useful for testing or when major structural changes occur.
+    """
+    global _path_cache, _shard_cache
+    _path_cache.clear()
+    _shard_cache.clear()
+
+
+def get_cache_stats() -> Dict[str, int]:
+    """Get statistics about cache usage.
+    
+    Returns:
+        Dictionary with cache statistics
+    """
+    return {
+        "path_cache_size": len(_path_cache),
+        "shard_cache_size": len(_shard_cache),
+        "max_cache_size": _MAX_CACHE_SIZE
+    }

cinchdb/core/tenant_activation.py

@@ -0,0 +1,216 @@
+"""Tenant activation and caching.
+
+This module handles local caching of tenant databases with
+LRU eviction and TTL management. WAL streaming is handled
+at the api-server layer, not in the core library.
+"""
+
+import os
+import sqlite3
+import hashlib
+from pathlib import Path
+from typing import Optional, Dict, Any
+from datetime import datetime, timedelta
+
+from cinchdb.core.connection import DatabaseConnection
+
+
+class TenantCache:
+    """Manages local cache of activated tenant databases.
+    
+    This implements an LRU cache with TTL for tenant SQLite files.
+    Tenants are activated on-demand from WAL replay and cached locally.
+    """
+    
+    def __init__(self, cache_dir: Optional[Path] = None, max_size_gb: float = 10.0):
+        """Initialize tenant cache.
+        
+        Args:
+            cache_dir: Directory for cached tenant databases
+            max_size_gb: Maximum cache size in GB before eviction
+        """
+        self.cache_dir = cache_dir or Path("/var/cache/cinchdb/tenants")
+        self.cache_dir.mkdir(parents=True, exist_ok=True)
+        self.max_size_bytes = int(max_size_gb * 1024 * 1024 * 1024)
+        
+        # Track cached tenants: tenant_key -> (path, version, last_access)
+        self.cache_index: Dict[str, Dict[str, Any]] = {}
+    
+    def _get_cache_path(self, database: str, branch: str, tenant: str) -> Path:
+        """Get cache path for a tenant database.
+        
+        Uses sharding to avoid too many files in one directory.
+        """
+        # Create tenant key
+        tenant_key = f"{database}_{branch}_{tenant}"
+        
+        # Calculate shard (2-level sharding like production)
+        hash_val = hashlib.sha256(tenant_key.encode()).hexdigest()
+        shard1 = hash_val[:2]
+        shard2 = hash_val[2:4]
+        
+        # Build path
+        cache_path = self.cache_dir / database / branch / shard1 / shard2
+        cache_path.mkdir(parents=True, exist_ok=True)
+        
+        return cache_path / f"{tenant}.db"
+    
+    def get_tenant_connection(
+        self,
+        database: str,
+        branch: str,
+        tenant: str,
+        wal_capture=None
+    ) -> DatabaseConnection:
+        """Get a database connection for a tenant.
+        
+        If the tenant is not cached, activates it from WAL replay.
+        
+        Args:
+            database: Database name
+            branch: Branch name
+            tenant: Tenant name
+            wal_capture: Optional WAL capture for streaming
+            
+        Returns:
+            DatabaseConnection to the tenant database
+        """
+        tenant_key = f"{database}_{branch}_{tenant}"
+        cache_path = self._get_cache_path(database, branch, tenant)
+        
+        # Check if already cached
+        if cache_path.exists():
+            # Update last access time
+            if tenant_key in self.cache_index:
+                self.cache_index[tenant_key]["last_access"] = datetime.now()
+            
+            return DatabaseConnection(cache_path, wal_capture=wal_capture)
+        
+        # Not cached - create empty database
+        # WAL streaming/activation happens at api-server layer if enabled
+        return self._create_empty_database(cache_path, wal_capture)
+    
+    def _create_empty_database(
+        self,
+        cache_path: Path,
+        wal_capture=None
+    ) -> DatabaseConnection:
+        """Create an empty database (for lazy tenants or non-streaming mode).
+        
+        Args:
+            cache_path: Path where database should be created
+            wal_capture: Optional WAL capture
+            
+        Returns:
+            DatabaseConnection to the new database
+        """
+        # Create empty SQLite database
+        conn = sqlite3.connect(str(cache_path))
+        
+        # Set up WAL mode
+        conn.execute("PRAGMA journal_mode = WAL")
+        conn.execute("PRAGMA synchronous = NORMAL")
+        conn.commit()
+        conn.close()
+        
+        return DatabaseConnection(cache_path, wal_capture=wal_capture)
+    
+    
+    def _evict_if_needed(self):
+        """Evict least recently used tenants if cache is too large."""
+        # Calculate total cache size
+        total_size = sum(
+            info["size_bytes"] for info in self.cache_index.values()
+        )
+        
+        if total_size <= self.max_size_bytes:
+            return  # Cache is within limits
+        
+        # Sort by last access time (LRU)
+        sorted_tenants = sorted(
+            self.cache_index.items(),
+            key=lambda x: x[1]["last_access"]
+        )
+        
+        # Evict until under limit
+        for tenant_key, info in sorted_tenants:
+            if total_size <= self.max_size_bytes:
+                break
+            
+            # Delete cached database
+            cache_path = info["path"]
+            if cache_path.exists():
+                # Delete SQLite files (main, WAL, SHM)
+                for suffix in ["", "-wal", "-shm"]:
+                    file_path = Path(str(cache_path) + suffix)
+                    if file_path.exists():
+                        file_path.unlink()
+                
+                print(f"Evicted cached tenant: {tenant_key}")
+            
+            # Remove from index
+            total_size -= info["size_bytes"]
+            del self.cache_index[tenant_key]
+    
+    def invalidate_tenant(self, database: str, branch: str, tenant: str):
+        """Invalidate a cached tenant (force re-activation on next access).
+        
+        Args:
+            database: Database name
+            branch: Branch name
+            tenant: Tenant name
+        """
+        tenant_key = f"{database}_{branch}_{tenant}"
+        
+        if tenant_key in self.cache_index:
+            info = self.cache_index[tenant_key]
+            cache_path = info["path"]
+            
+            # Delete cached files
+            if cache_path.exists():
+                for suffix in ["", "-wal", "-shm"]:
+                    file_path = Path(str(cache_path) + suffix)
+                    if file_path.exists():
+                        file_path.unlink()
+            
+            # Remove from index
+            del self.cache_index[tenant_key]
+            print(f"Invalidated cached tenant: {tenant_key}")
+    
+    def get_cache_stats(self) -> Dict[str, Any]:
+        """Get cache statistics.
+        
+        Returns:
+            Dictionary with cache stats
+        """
+        total_size = sum(
+            info["size_bytes"] for info in self.cache_index.values()
+        )
+        
+        return {
+            "cached_tenants": len(self.cache_index),
+            "total_size_bytes": total_size,
+            "total_size_mb": round(total_size / (1024 * 1024), 2),
+            "max_size_gb": self.max_size_bytes / (1024 * 1024 * 1024),
+            "cache_dir": str(self.cache_dir)
+        }
+
+
+# Global cache instance (singleton)
+_tenant_cache: Optional[TenantCache] = None
+
+
+def get_tenant_cache() -> TenantCache:
+    """Get the global tenant cache instance.
+    
+    Returns:
+        TenantCache singleton
+    """
+    global _tenant_cache
+    
+    if _tenant_cache is None:
+        cache_dir = os.getenv("TENANT_CACHE_DIR", "/var/cache/cinchdb")
+        cache_size_gb = float(os.getenv("TENANT_CACHE_SIZE_GB", "10.0"))
+        _tenant_cache = TenantCache(Path(cache_dir), cache_size_gb)
+    
+    return _tenant_cache

cinchdb/infrastructure/metadata_db.py

@@ -37,6 +37,7 @@ class MetadataDB:
         self.conn.execute("PRAGMA foreign_keys = ON")
         self.conn.execute("PRAGMA journal_mode = WAL")  # Better concurrency
     
+    
     def _create_tables(self):
         """Create the metadata tables if they don't exist."""
         with self.conn:
@@ -47,6 +48,9 @@ class MetadataDB:
                     name TEXT NOT NULL UNIQUE,
                     description TEXT,
                     materialized BOOLEAN DEFAULT FALSE,
+                    maintenance_mode BOOLEAN DEFAULT FALSE,
+                    maintenance_reason TEXT,
+                    maintenance_started_at TIMESTAMP,
                     metadata JSON,
                     created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                     updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
@@ -62,6 +66,9 @@ class MetadataDB:
                     parent_branch TEXT,
                     schema_version TEXT,
                     materialized BOOLEAN DEFAULT FALSE,
+                    maintenance_mode BOOLEAN DEFAULT FALSE,
+                    maintenance_reason TEXT,
+                    maintenance_started_at TIMESTAMP,
                     metadata JSON,
                     created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                     updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
@@ -116,9 +123,7 @@ class MetadataDB:
                 CREATE INDEX IF NOT EXISTS idx_tenants_shard 
                 ON tenants(shard)
             """)
-            
-            # Add cdc_enabled field to branches table for plugin use
-            # Note: This field is managed by plugins (like bdhcnic), not core CinchDB
+
             try:
                 self.conn.execute("""
                     ALTER TABLE branches ADD COLUMN cdc_enabled BOOLEAN DEFAULT FALSE
@@ -376,6 +381,94 @@ class MetadataDB:
             
             return len(tenants)
     
+    # Maintenance Mode Methods
+    
+    def set_database_maintenance(self, database_name: str, enabled: bool, reason: Optional[str] = None) -> None:
+        """Set maintenance mode for a database."""
+        with self.conn:
+            if enabled:
+                self.conn.execute("""
+                    UPDATE databases 
+                    SET maintenance_mode = TRUE, 
+                        maintenance_reason = ?, 
+                        maintenance_started_at = CURRENT_TIMESTAMP 
+                    WHERE name = ?
+                """, (reason, database_name))
+            else:
+                self.conn.execute("""
+                    UPDATE databases 
+                    SET maintenance_mode = FALSE, 
+                        maintenance_reason = NULL, 
+                        maintenance_started_at = NULL 
+                    WHERE name = ?
+                """, (database_name,))
+    
+    def set_branch_maintenance(self, database_name: str, branch_name: str, enabled: bool, reason: Optional[str] = None) -> None:
+        """Set maintenance mode for a branch."""
+        with self.conn:
+            if enabled:
+                self.conn.execute("""
+                    UPDATE branches 
+                    SET maintenance_mode = TRUE, 
+                        maintenance_reason = ?, 
+                        maintenance_started_at = CURRENT_TIMESTAMP 
+                    WHERE database_id = (SELECT id FROM databases WHERE name = ?) 
+                    AND name = ?
+                """, (reason, database_name, branch_name))
+            else:
+                self.conn.execute("""
+                    UPDATE branches 
+                    SET maintenance_mode = FALSE, 
+                        maintenance_reason = NULL, 
+                        maintenance_started_at = NULL 
+                    WHERE database_id = (SELECT id FROM databases WHERE name = ?) 
+                    AND name = ?
+                """, (database_name, branch_name))
+    
+    def is_database_in_maintenance(self, database_name: str) -> bool:
+        """Check if database is in maintenance mode."""
+        cursor = self.conn.execute("""
+            SELECT maintenance_mode FROM databases WHERE name = ?
+        """, (database_name,))
+        row = cursor.fetchone()
+        return bool(row and row['maintenance_mode'])
+    
+    def is_branch_in_maintenance(self, database_name: str, branch_name: str) -> bool:
+        """Check if branch is in maintenance mode."""
+        cursor = self.conn.execute("""
+            SELECT maintenance_mode FROM branches 
+            WHERE database_id = (SELECT id FROM databases WHERE name = ?) 
+            AND name = ?
+        """, (database_name, branch_name))
+        row = cursor.fetchone()
+        return bool(row and row['maintenance_mode'])
+    
+    def get_maintenance_info(self, database_name: str, branch_name: Optional[str] = None) -> Optional[Dict[str, Any]]:
+        """Get maintenance mode information."""
+        if branch_name:
+            # Check branch maintenance
+            cursor = self.conn.execute("""
+                SELECT maintenance_mode, maintenance_reason, maintenance_started_at 
+                FROM branches 
+                WHERE database_id = (SELECT id FROM databases WHERE name = ?) 
+                AND name = ?
+            """, (database_name, branch_name))
+        else:
+            # Check database maintenance
+            cursor = self.conn.execute("""
+                SELECT maintenance_mode, maintenance_reason, maintenance_started_at 
+                FROM databases WHERE name = ?
+            """, (database_name,))
+        
+        row = cursor.fetchone()
+        if row and row['maintenance_mode']:
+            return {
+                'enabled': True,
+                'reason': row['maintenance_reason'],
+                'started_at': row['maintenance_started_at']
+            }
+        return None
+    
     def close(self):
         """Close the database connection."""
         if self.conn: