cinchdb 0.1.15__py3-none-any.whl → 0.1.17__py3-none-any.whl

cinchdb/config.py

@@ -134,17 +134,8 @@ class Config:
         with open(self.config_path, "w") as f:
             toml.dump(config_dict, f)
 
-    def init_project(self) -> ProjectConfig:
-        """Initialize a new CinchDB project with default configuration.
-
-        This method now delegates to the ProjectInitializer for the actual
-        initialization logic.
-        """
-        from cinchdb.core.initializer import ProjectInitializer
-
-        initializer = ProjectInitializer(self.project_dir)
-        config = initializer.init_project()
+    @property
+    def base_dir(self) -> Path:
+        """Get the base project directory."""
+        return self.project_dir
 
-        # Load the config into this instance
-        self._config = config
-        return config

cinchdb/core/connection.py

@@ -1,8 +1,9 @@
 """SQLite connection management for CinchDB."""
 
+import os
 import sqlite3
 from pathlib import Path
-from typing import Optional, Dict, List
+from typing import Optional, Dict, List, Any
 from contextlib import contextmanager
 from datetime import datetime
 

cinchdb/core/database.py

@@ -1,11 +1,13 @@
 """Unified database connection interface for CinchDB."""
 
+import os
 from pathlib import Path
 from typing import List, Dict, Any, Optional, TYPE_CHECKING
 
 from cinchdb.models import Column, Change
 from cinchdb.core.path_utils import get_project_root
 from cinchdb.utils import validate_query_safe
+from cinchdb.infrastructure.metadata_connection_pool import get_metadata_db
 
 if TYPE_CHECKING:
     from cinchdb.managers.table import TableManager
@@ -78,7 +80,7 @@ class CinchDB:
         self.database = database
         self.branch = branch
         self.tenant = tenant
-
+        
         # Determine connection type
         if project_dir is not None:
             # Local connection
@@ -120,16 +122,26 @@ class CinchDB:
             return
             
         # Check if this is a lazy database using metadata DB
-        from cinchdb.infrastructure.metadata_db import MetadataDB
-        
-        with MetadataDB(self.project_dir) as metadata_db:
-            db_info = metadata_db.get_database(self.database)
+        metadata_db = get_metadata_db(self.project_dir)
+        db_info = metadata_db.get_database(self.database)
         
         if db_info and not db_info['materialized']:
             # Database exists in metadata but not materialized
             from cinchdb.core.initializer import ProjectInitializer
             initializer = ProjectInitializer(self.project_dir)
             initializer.materialize_database(self.database)
+    
+    def get_connection(self, db_path) -> "DatabaseConnection":
+        """Get a database connection.
+        
+        Args:
+            db_path: Path to database file
+            
+        Returns:
+            DatabaseConnection instance
+        """
+        from cinchdb.core.connection import DatabaseConnection
+        return DatabaseConnection(db_path)
 
     @property
     def session(self):

cinchdb/core/maintenance_utils.py

@@ -0,0 +1,43 @@
+"""Maintenance utilities for CinchDB operations."""
+
+from pathlib import Path
+from cinchdb.infrastructure.metadata_connection_pool import get_metadata_db
+
+
+class MaintenanceError(Exception):
+    """Exception raised when operation blocked by maintenance mode."""
+    pass
+
+
+def check_maintenance_mode(project_root: Path, database: str, branch: str = None) -> None:
+    """Check if database or branch is in maintenance mode and raise error if so.
+    
+    Args:
+        project_root: Path to project root
+        database: Database name  
+        branch: Branch name (optional)
+        
+    Raises:
+        MaintenanceError: If database or branch is in maintenance mode
+    """
+    try:
+        metadata_db = get_metadata_db(project_root)
+        
+        # Check database-level maintenance
+        if metadata_db.is_database_in_maintenance(database):
+            info = metadata_db.get_maintenance_info(database)
+            reason = info.get("reason", "Database maintenance in progress") if info else "Database maintenance in progress"
+            raise MaintenanceError(f"Database '{database}' is in maintenance mode: {reason}")
+        
+        # Check branch-level maintenance if branch specified
+        if branch and metadata_db.is_branch_in_maintenance(database, branch):
+            info = metadata_db.get_maintenance_info(database, branch)
+            reason = info.get("reason", "Branch maintenance in progress") if info else "Branch maintenance in progress"
+            raise MaintenanceError(f"Branch '{database}/{branch}' is in maintenance mode: {reason}")
+            
+    except MaintenanceError:
+        raise  # Re-raise maintenance errors
+    except Exception:
+        # If we can't check maintenance status, allow the operation to proceed
+        # This prevents maintenance check failures from blocking normal operations
+        pass

cinchdb/core/path_utils.py

@@ -2,6 +2,7 @@
 
 from pathlib import Path
 from typing import List
+from cinchdb.infrastructure.metadata_connection_pool import get_metadata_db
 
 
 def get_project_root(start_path: Path) -> Path:
@@ -119,10 +120,9 @@ def list_databases(project_root: Path) -> List[str]:
     if not metadata_db_path.exists():
         return []
     
-    from cinchdb.infrastructure.metadata_db import MetadataDB
-    with MetadataDB(project_root) as metadata_db:
-        db_records = metadata_db.list_databases()
-        return sorted(record['name'] for record in db_records)
+    metadata_db = get_metadata_db(project_root)
+    db_records = metadata_db.list_databases()
+    return sorted(record['name'] for record in db_records)
 
 
 def list_branches(project_root: Path, database: str) -> List[str]:
@@ -139,13 +139,12 @@ def list_branches(project_root: Path, database: str) -> List[str]:
     if not metadata_db_path.exists():
         return []
     
-    from cinchdb.infrastructure.metadata_db import MetadataDB
-    with MetadataDB(project_root) as metadata_db:
-        db_info = metadata_db.get_database(database)
-        if not db_info:
-            return []
-        branch_records = metadata_db.list_branches(db_info['id'])
-        return sorted(record['name'] for record in branch_records)
+    metadata_db = get_metadata_db(project_root)
+    db_info = metadata_db.get_database(database)
+    if not db_info:
+        return []
+    branch_records = metadata_db.list_branches(db_info['id'])
+    return sorted(record['name'] for record in branch_records)
 
 
 def list_tenants(project_root: Path, database: str, branch: str) -> List[str]:
@@ -163,13 +162,12 @@ def list_tenants(project_root: Path, database: str, branch: str) -> List[str]:
     if not metadata_db_path.exists():
         return []
     
-    from cinchdb.infrastructure.metadata_db import MetadataDB
-    with MetadataDB(project_root) as metadata_db:
-        db_info = metadata_db.get_database(database)
-        if not db_info:
-            return []
-        branch_info = metadata_db.get_branch(db_info['id'], branch)
-        if not branch_info:
-            return []
-        tenant_records = metadata_db.list_tenants(branch_info['id'])
-        return sorted(record['name'] for record in tenant_records)
+    metadata_db = get_metadata_db(project_root)
+    db_info = metadata_db.get_database(database)
+    if not db_info:
+        return []
+    branch_info = metadata_db.get_branch(db_info['id'], branch)
+    if not branch_info:
+        return []
+    tenant_records = metadata_db.list_tenants(branch_info['id'])
+    return sorted(record['name'] for record in tenant_records)

cinchdb/core/tenant_activation.py

@@ -0,0 +1,216 @@
+"""Tenant activation and caching.
+
+This module handles local caching of tenant databases with
+LRU eviction and TTL management. WAL streaming is handled
+at the api-server layer, not in the core library.
+"""
+
+import os
+import sqlite3
+import hashlib
+from pathlib import Path
+from typing import Optional, Dict, Any
+from datetime import datetime, timedelta
+
+from cinchdb.core.connection import DatabaseConnection
+
+
+class TenantCache:
+    """Manages local cache of activated tenant databases.
+    
+    This implements an LRU cache with TTL for tenant SQLite files.
+    Tenants are activated on-demand from WAL replay and cached locally.
+    """
+    
+    def __init__(self, cache_dir: Optional[Path] = None, max_size_gb: float = 10.0):
+        """Initialize tenant cache.
+        
+        Args:
+            cache_dir: Directory for cached tenant databases
+            max_size_gb: Maximum cache size in GB before eviction
+        """
+        self.cache_dir = cache_dir or Path("/var/cache/cinchdb/tenants")
+        self.cache_dir.mkdir(parents=True, exist_ok=True)
+        self.max_size_bytes = int(max_size_gb * 1024 * 1024 * 1024)
+        
+        # Track cached tenants: tenant_key -> (path, version, last_access)
+        self.cache_index: Dict[str, Dict[str, Any]] = {}
+    
+    def _get_cache_path(self, database: str, branch: str, tenant: str) -> Path:
+        """Get cache path for a tenant database.
+        
+        Uses sharding to avoid too many files in one directory.
+        """
+        # Create tenant key
+        tenant_key = f"{database}_{branch}_{tenant}"
+        
+        # Calculate shard (2-level sharding like production)
+        hash_val = hashlib.sha256(tenant_key.encode()).hexdigest()
+        shard1 = hash_val[:2]
+        shard2 = hash_val[2:4]
+        
+        # Build path
+        cache_path = self.cache_dir / database / branch / shard1 / shard2
+        cache_path.mkdir(parents=True, exist_ok=True)
+        
+        return cache_path / f"{tenant}.db"
+    
+    def get_tenant_connection(
+        self,
+        database: str,
+        branch: str,
+        tenant: str,
+        wal_capture=None
+    ) -> DatabaseConnection:
+        """Get a database connection for a tenant.
+        
+        If the tenant is not cached, activates it from WAL replay.
+        
+        Args:
+            database: Database name
+            branch: Branch name
+            tenant: Tenant name
+            wal_capture: Optional WAL capture for streaming
+            
+        Returns:
+            DatabaseConnection to the tenant database
+        """
+        tenant_key = f"{database}_{branch}_{tenant}"
+        cache_path = self._get_cache_path(database, branch, tenant)
+        
+        # Check if already cached
+        if cache_path.exists():
+            # Update last access time
+            if tenant_key in self.cache_index:
+                self.cache_index[tenant_key]["last_access"] = datetime.now()
+            
+            return DatabaseConnection(cache_path, wal_capture=wal_capture)
+        
+        # Not cached - create empty database
+        # WAL streaming/activation happens at api-server layer if enabled
+        return self._create_empty_database(cache_path, wal_capture)
+    
+    def _create_empty_database(
+        self,
+        cache_path: Path,
+        wal_capture=None
+    ) -> DatabaseConnection:
+        """Create an empty database (for lazy tenants or non-streaming mode).
+        
+        Args:
+            cache_path: Path where database should be created
+            wal_capture: Optional WAL capture
+            
+        Returns:
+            DatabaseConnection to the new database
+        """
+        # Create empty SQLite database
+        conn = sqlite3.connect(str(cache_path))
+        
+        # Set up WAL mode
+        conn.execute("PRAGMA journal_mode = WAL")
+        conn.execute("PRAGMA synchronous = NORMAL")
+        conn.commit()
+        conn.close()
+        
+        return DatabaseConnection(cache_path, wal_capture=wal_capture)
+    
+    
+    def _evict_if_needed(self):
+        """Evict least recently used tenants if cache is too large."""
+        # Calculate total cache size
+        total_size = sum(
+            info["size_bytes"] for info in self.cache_index.values()
+        )
+        
+        if total_size <= self.max_size_bytes:
+            return  # Cache is within limits
+        
+        # Sort by last access time (LRU)
+        sorted_tenants = sorted(
+            self.cache_index.items(),
+            key=lambda x: x[1]["last_access"]
+        )
+        
+        # Evict until under limit
+        for tenant_key, info in sorted_tenants:
+            if total_size <= self.max_size_bytes:
+                break
+            
+            # Delete cached database
+            cache_path = info["path"]
+            if cache_path.exists():
+                # Delete SQLite files (main, WAL, SHM)
+                for suffix in ["", "-wal", "-shm"]:
+                    file_path = Path(str(cache_path) + suffix)
+                    if file_path.exists():
+                        file_path.unlink()
+                
+                print(f"Evicted cached tenant: {tenant_key}")
+            
+            # Remove from index
+            total_size -= info["size_bytes"]
+            del self.cache_index[tenant_key]
+    
+    def invalidate_tenant(self, database: str, branch: str, tenant: str):
+        """Invalidate a cached tenant (force re-activation on next access).
+        
+        Args:
+            database: Database name
+            branch: Branch name
+            tenant: Tenant name
+        """
+        tenant_key = f"{database}_{branch}_{tenant}"
+        
+        if tenant_key in self.cache_index:
+            info = self.cache_index[tenant_key]
+            cache_path = info["path"]
+            
+            # Delete cached files
+            if cache_path.exists():
+                for suffix in ["", "-wal", "-shm"]:
+                    file_path = Path(str(cache_path) + suffix)
+                    if file_path.exists():
+                        file_path.unlink()
+            
+            # Remove from index
+            del self.cache_index[tenant_key]
+            print(f"Invalidated cached tenant: {tenant_key}")
+    
+    def get_cache_stats(self) -> Dict[str, Any]:
+        """Get cache statistics.
+        
+        Returns:
+            Dictionary with cache stats
+        """
+        total_size = sum(
+            info["size_bytes"] for info in self.cache_index.values()
+        )
+        
+        return {
+            "cached_tenants": len(self.cache_index),
+            "total_size_bytes": total_size,
+            "total_size_mb": round(total_size / (1024 * 1024), 2),
+            "max_size_gb": self.max_size_bytes / (1024 * 1024 * 1024),
+            "cache_dir": str(self.cache_dir)
+        }
+
+
+# Global cache instance (singleton)
+_tenant_cache: Optional[TenantCache] = None
+
+
+def get_tenant_cache() -> TenantCache:
+    """Get the global tenant cache instance.
+    
+    Returns:
+        TenantCache singleton
+    """
+    global _tenant_cache
+    
+    if _tenant_cache is None:
+        cache_dir = os.getenv("TENANT_CACHE_DIR", "/var/cache/cinchdb")
+        cache_size_gb = float(os.getenv("TENANT_CACHE_SIZE_GB", "10.0"))
+        _tenant_cache = TenantCache(Path(cache_dir), cache_size_gb)
+    
+    return _tenant_cache

cinchdb/infrastructure/metadata_db.py

@@ -37,6 +37,7 @@ class MetadataDB:
         self.conn.execute("PRAGMA foreign_keys = ON")
         self.conn.execute("PRAGMA journal_mode = WAL")  # Better concurrency
     
+    
     def _create_tables(self):
         """Create the metadata tables if they don't exist."""
         with self.conn:
@@ -47,6 +48,9 @@ class MetadataDB:
                     name TEXT NOT NULL UNIQUE,
                     description TEXT,
                     materialized BOOLEAN DEFAULT FALSE,
+                    maintenance_mode BOOLEAN DEFAULT FALSE,
+                    maintenance_reason TEXT,
+                    maintenance_started_at TIMESTAMP,
                     metadata JSON,
                     created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                     updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
@@ -62,6 +66,9 @@ class MetadataDB:
                     parent_branch TEXT,
                     schema_version TEXT,
                     materialized BOOLEAN DEFAULT FALSE,
+                    maintenance_mode BOOLEAN DEFAULT FALSE,
+                    maintenance_reason TEXT,
+                    maintenance_started_at TIMESTAMP,
                     metadata JSON,
                     created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                     updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
@@ -116,9 +123,7 @@ class MetadataDB:
                 CREATE INDEX IF NOT EXISTS idx_tenants_shard 
                 ON tenants(shard)
             """)
-            
-            # Add cdc_enabled field to branches table for plugin use
-            # Note: This field is managed by plugins (like bdhcnic), not core CinchDB
+
             try:
                 self.conn.execute("""
                     ALTER TABLE branches ADD COLUMN cdc_enabled BOOLEAN DEFAULT FALSE
@@ -376,6 +381,94 @@ class MetadataDB:
             
             return len(tenants)
     
+    # Maintenance Mode Methods
+    
+    def set_database_maintenance(self, database_name: str, enabled: bool, reason: Optional[str] = None) -> None:
+        """Set maintenance mode for a database."""
+        with self.conn:
+            if enabled:
+                self.conn.execute("""
+                    UPDATE databases 
+                    SET maintenance_mode = TRUE, 
+                        maintenance_reason = ?, 
+                        maintenance_started_at = CURRENT_TIMESTAMP 
+                    WHERE name = ?
+                """, (reason, database_name))
+            else:
+                self.conn.execute("""
+                    UPDATE databases 
+                    SET maintenance_mode = FALSE, 
+                        maintenance_reason = NULL, 
+                        maintenance_started_at = NULL 
+                    WHERE name = ?
+                """, (database_name,))
+    
+    def set_branch_maintenance(self, database_name: str, branch_name: str, enabled: bool, reason: Optional[str] = None) -> None:
+        """Set maintenance mode for a branch."""
+        with self.conn:
+            if enabled:
+                self.conn.execute("""
+                    UPDATE branches 
+                    SET maintenance_mode = TRUE, 
+                        maintenance_reason = ?, 
+                        maintenance_started_at = CURRENT_TIMESTAMP 
+                    WHERE database_id = (SELECT id FROM databases WHERE name = ?) 
+                    AND name = ?
+                """, (reason, database_name, branch_name))
+            else:
+                self.conn.execute("""
+                    UPDATE branches 
+                    SET maintenance_mode = FALSE, 
+                        maintenance_reason = NULL, 
+                        maintenance_started_at = NULL 
+                    WHERE database_id = (SELECT id FROM databases WHERE name = ?) 
+                    AND name = ?
+                """, (database_name, branch_name))
+    
+    def is_database_in_maintenance(self, database_name: str) -> bool:
+        """Check if database is in maintenance mode."""
+        cursor = self.conn.execute("""
+            SELECT maintenance_mode FROM databases WHERE name = ?
+        """, (database_name,))
+        row = cursor.fetchone()
+        return bool(row and row['maintenance_mode'])
+    
+    def is_branch_in_maintenance(self, database_name: str, branch_name: str) -> bool:
+        """Check if branch is in maintenance mode."""
+        cursor = self.conn.execute("""
+            SELECT maintenance_mode FROM branches 
+            WHERE database_id = (SELECT id FROM databases WHERE name = ?) 
+            AND name = ?
+        """, (database_name, branch_name))
+        row = cursor.fetchone()
+        return bool(row and row['maintenance_mode'])
+    
+    def get_maintenance_info(self, database_name: str, branch_name: Optional[str] = None) -> Optional[Dict[str, Any]]:
+        """Get maintenance mode information."""
+        if branch_name:
+            # Check branch maintenance
+            cursor = self.conn.execute("""
+                SELECT maintenance_mode, maintenance_reason, maintenance_started_at 
+                FROM branches 
+                WHERE database_id = (SELECT id FROM databases WHERE name = ?) 
+                AND name = ?
+            """, (database_name, branch_name))
+        else:
+            # Check database maintenance
+            cursor = self.conn.execute("""
+                SELECT maintenance_mode, maintenance_reason, maintenance_started_at 
+                FROM databases WHERE name = ?
+            """, (database_name,))
+        
+        row = cursor.fetchone()
+        if row and row['maintenance_mode']:
+            return {
+                'enabled': True,
+                'reason': row['maintenance_reason'],
+                'started_at': row['maintenance_started_at']
+            }
+        return None
+    
     def close(self):
         """Close the database connection."""
         if self.conn:

cinchdb/managers/change_applier.py

@@ -11,6 +11,7 @@ from cinchdb.managers.change_tracker import ChangeTracker
 from cinchdb.managers.tenant import TenantManager
 from cinchdb.core.connection import DatabaseConnection
 from cinchdb.core.path_utils import get_tenant_db_path, get_branch_path
+from cinchdb.infrastructure.metadata_connection_pool import get_metadata_db
 
 logger = logging.getLogger(__name__)
 
@@ -388,21 +389,17 @@ class ChangeApplier:
 
     def _enter_maintenance_mode(self) -> None:
         """Enter maintenance mode to block writes during schema changes."""
-        # Create a maintenance mode file that all connections can check
-        branch_path = get_branch_path(self.project_root, self.database, self.branch)
-        maintenance_file = branch_path / ".maintenance_mode"
-
-        with open(maintenance_file, "w") as f:
-            import json
-
-            json.dump(
-                {
-                    "active": True,
-                    "reason": "Schema update in progress",
-                    "started_at": datetime.now().isoformat(),
-                },
-                f,
+        try:
+            metadata_db = get_metadata_db(self.project_root)
+            metadata_db.set_branch_maintenance(
+                self.database, 
+                self.branch, 
+                True, 
+                "Schema update in progress"
             )
+        except Exception as e:
+            logger.error(f"Failed to enter maintenance mode: {e}")
+            # Continue anyway - we'll try to proceed with the schema update
 
         # Give time for any in-flight writes to complete
         # Can be disabled for tests via environment variable
@@ -414,11 +411,11 @@ class ChangeApplier:
 
     def _exit_maintenance_mode(self) -> None:
         """Exit maintenance mode to allow writes again."""
-        branch_path = get_branch_path(self.project_root, self.database, self.branch)
-        maintenance_file = branch_path / ".maintenance_mode"
-
-        if maintenance_file.exists():
-            maintenance_file.unlink()
+        try:
+            metadata_db = get_metadata_db(self.project_root)
+            metadata_db.set_branch_maintenance(self.database, self.branch, False)
+        except Exception as e:
+            logger.error(f"Failed to exit maintenance mode: {e}")
 
     def is_in_maintenance_mode(self) -> bool:
         """Check if branch is in maintenance mode.
@@ -426,6 +423,8 @@ class ChangeApplier:
         Returns:
             True if in maintenance mode, False otherwise
         """
-        branch_path = get_branch_path(self.project_root, self.database, self.branch)
-        maintenance_file = branch_path / ".maintenance_mode"
-        return maintenance_file.exists()
+        try:
+            metadata_db = get_metadata_db(self.project_root)
+            return metadata_db.is_branch_in_maintenance(self.database, self.branch)
+        except Exception:
+            return False

cinchdb/managers/column.py

@@ -6,7 +6,7 @@ from typing import List, Any, Optional
 from cinchdb.models import Column, Change, ChangeType
 from cinchdb.core.connection import DatabaseConnection
 from cinchdb.core.path_utils import get_tenant_db_path
-from cinchdb.core.maintenance import check_maintenance_mode
+from cinchdb.core.maintenance_utils import check_maintenance_mode
 from cinchdb.managers.change_tracker import ChangeTracker
 from cinchdb.managers.table import TableManager
 

cinchdb/managers/data.py

@@ -9,7 +9,7 @@ from pydantic import BaseModel
 
 from cinchdb.core.connection import DatabaseConnection
 from cinchdb.core.path_utils import get_tenant_db_path
-from cinchdb.core.maintenance import check_maintenance_mode
+from cinchdb.core.maintenance_utils import check_maintenance_mode
 from cinchdb.managers.table import TableManager
 from cinchdb.managers.query import QueryManager
 

cinchdb/managers/table.py

@@ -9,7 +9,7 @@ if TYPE_CHECKING:
     from cinchdb.models import Index
 from cinchdb.core.connection import DatabaseConnection
 from cinchdb.core.path_utils import get_tenant_db_path
-from cinchdb.core.maintenance import check_maintenance_mode
+from cinchdb.core.maintenance_utils import check_maintenance_mode
 from cinchdb.managers.change_tracker import ChangeTracker
 from cinchdb.managers.tenant import TenantManager
 

cinchdb/managers/tenant.py

@@ -15,7 +15,7 @@ from cinchdb.core.path_utils import (
     list_tenants,
 )
 from cinchdb.core.connection import DatabaseConnection
-from cinchdb.core.maintenance import check_maintenance_mode
+from cinchdb.core.maintenance_utils import check_maintenance_mode
 from cinchdb.utils.name_validator import validate_name
 from cinchdb.infrastructure.metadata_db import MetadataDB
 from cinchdb.infrastructure.metadata_connection_pool import get_metadata_db

cinchdb/managers/view.py

@@ -6,7 +6,7 @@ from typing import List
 from cinchdb.models import View, Change, ChangeType
 from cinchdb.core.connection import DatabaseConnection
 from cinchdb.core.path_utils import get_tenant_db_path
-from cinchdb.core.maintenance import check_maintenance_mode
+from cinchdb.core.maintenance_utils import check_maintenance_mode
 from cinchdb.managers.change_tracker import ChangeTracker