cinchdb 0.1.10__py3-none-any.whl → 0.1.11__py3-none-any.whl

cinchdb/core/initializer.py

@@ -1,12 +1,29 @@
 """Project initialization for CinchDB."""
 
+import hashlib
 import json
+import uuid
 from datetime import datetime, timezone
 from pathlib import Path
 from typing import Optional
 
 from cinchdb.core.connection import DatabaseConnection
 from cinchdb.config import ProjectConfig
+from cinchdb.infrastructure.metadata_db import MetadataDB
+from cinchdb.infrastructure.metadata_connection_pool import get_metadata_db
+
+
+def _calculate_shard(tenant_name: str) -> str:
+    """Calculate the shard directory for a tenant using SHA256 hash.
+    
+    Args:
+        tenant_name: Name of the tenant
+        
+    Returns:
+        Two-character hex string (e.g., "a0", "ff")
+    """
+    hash_val = hashlib.sha256(tenant_name.encode('utf-8')).hexdigest()
+    return hash_val[:2]
 
 
 class ProjectInitializer:
@@ -21,6 +38,14 @@ class ProjectInitializer:
         self.project_dir = Path(project_dir) if project_dir else Path.cwd()
         self.config_dir = self.project_dir / ".cinchdb"
         self.config_path = self.config_dir / "config.toml"
+        self._metadata_db = None
+    
+    @property
+    def metadata_db(self) -> MetadataDB:
+        """Get metadata database connection (lazy-initialized from pool)."""
+        if self._metadata_db is None:
+            self._metadata_db = get_metadata_db(self.project_dir)
+        return self._metadata_db
 
     def init_project(
         self, database_name: str = "main", branch_name: str = "main"
@@ -36,7 +61,13 @@ class ProjectInitializer:
 
         Raises:
             FileExistsError: If project already exists at the location
+            InvalidNameError: If database name is invalid
         """
+        from cinchdb.utils.name_validator import validate_name
+        
+        # Validate database name
+        validate_name(database_name, "database")
+        
         if self.config_path.exists():
             raise FileExistsError(f"Project already exists at {self.config_dir}")
 
@@ -48,9 +79,57 @@ class ProjectInitializer:
 
         # Save config
         self._save_config(config)
+        
+        # Add initial database to metadata (metadata_db property will auto-initialize)
+        database_id = str(uuid.uuid4())
+        self.metadata_db.create_database(
+            database_id, database_name, 
+            description="Initial database",
+            metadata={"initial_branch": branch_name}
+        )
+        
+        # Add initial branch to metadata
+        branch_id = str(uuid.uuid4())
+        self.metadata_db.create_branch(
+            branch_id, database_id, branch_name,
+            parent_branch=None,
+            schema_version="v1.0.0",
+            metadata={"created_at": datetime.now(timezone.utc).isoformat()}
+        )
 
-        # Create default database structure
-        self._create_database_structure(database_name, branch_name)
+        # Create default database structure (materialized by default for initial database)
+        self._create_database_structure(database_name, branch_name, create_tenant_files=True)
+        
+        # Mark as materialized since we created the structure
+        self.metadata_db.mark_database_materialized(database_id)
+        self.metadata_db.mark_branch_materialized(branch_id)
+        
+        # Also create main tenant in metadata
+        tenant_id = str(uuid.uuid4())
+        main_shard = _calculate_shard("main")
+        self.metadata_db.create_tenant(
+            tenant_id, branch_id, "main", main_shard,
+            metadata={"created_at": datetime.now(timezone.utc).isoformat()}
+        )
+        self.metadata_db.mark_tenant_materialized(tenant_id)
+        
+        # Create __empty__ tenant in metadata (for lazy tenant reads)
+        empty_tenant_id = str(uuid.uuid4())
+        empty_shard = _calculate_shard("__empty__")
+        self.metadata_db.create_tenant(
+            empty_tenant_id, branch_id, "__empty__", empty_shard,
+            metadata={
+                "system": True,
+                "description": "Template for lazy tenants",
+                "created_at": datetime.now(timezone.utc).isoformat()
+            }
+        )
+        self.metadata_db.mark_tenant_materialized(empty_tenant_id)
+        
+        # Create physical __empty__ tenant with schema from main
+        from cinchdb.managers.tenant import TenantManager
+        tenant_mgr = TenantManager(self.project_dir, database_name, branch_name)
+        tenant_mgr._ensure_empty_tenant()
 
         return config
 
@@ -59,6 +138,7 @@ class ProjectInitializer:
         database_name: str,
         branch_name: str = "main",
         description: Optional[str] = None,
+        lazy: bool = True,
     ) -> None:
         """Initialize a new database within an existing project.
 
@@ -66,26 +146,78 @@ class ProjectInitializer:
             database_name: Name for the database
             branch_name: Initial branch name (default: "main")
             description: Optional description for the database
+            lazy: If True, don't create actual database files until first use
 
         Raises:
             FileNotFoundError: If project doesn't exist
             FileExistsError: If database already exists
+            InvalidNameError: If database name is invalid
         """
+        from cinchdb.utils.name_validator import validate_name
+        
+        # Validate database name
+        validate_name(database_name, "database")
+        
         if not self.config_path.exists():
             raise FileNotFoundError(f"No CinchDB project found at {self.config_dir}")
-
-        db_path = self.config_dir / "databases" / database_name
-        if db_path.exists():
+        
+        # Check if database already exists in metadata
+        existing_db = self.metadata_db.get_database(database_name)
+        if existing_db:
             raise FileExistsError(f"Database '{database_name}' already exists")
 
-        # Create database structure
-        self._create_database_structure(database_name, branch_name, description)
+        # Create database ID
+        database_id = str(uuid.uuid4())
+        
+        # Create database in metadata
+        metadata = {
+            "description": description,
+            "initial_branch": branch_name,
+            "created_at": datetime.now(timezone.utc).isoformat(),
+        }
+        self.metadata_db.create_database(database_id, database_name, description, metadata)
+        
+        # Create initial branch in metadata
+        branch_id = str(uuid.uuid4())
+        self.metadata_db.create_branch(
+            branch_id, database_id, branch_name,
+            parent_branch=None,
+            schema_version="v1.0.0",
+            metadata={"created_at": datetime.now(timezone.utc).isoformat()}
+        )
+        
+        # Create main tenant entry in metadata (will be materialized if database is not lazy)
+        main_tenant_id = str(uuid.uuid4())
+        main_shard = _calculate_shard("main")
+        self.metadata_db.create_tenant(
+            main_tenant_id, branch_id, "main", main_shard,
+            metadata={"description": "Default tenant", "created_at": datetime.now(timezone.utc).isoformat()}
+        )
+        
+        # Create __empty__ tenant entry in metadata (lazy)
+        # This serves as a template for all lazy tenants in this branch
+        empty_tenant_id = str(uuid.uuid4())
+        empty_shard = _calculate_shard("__empty__")
+        self.metadata_db.create_tenant(
+            empty_tenant_id, branch_id, "__empty__", empty_shard,
+            metadata={"system": True, "description": "Template for lazy tenants"}
+        )
+
+        if not lazy:
+            # Create actual database structure
+            self._create_database_structure(database_name, branch_name, description)
+            
+            # Mark as materialized
+            self.metadata_db.mark_database_materialized(database_id)
+            self.metadata_db.mark_branch_materialized(branch_id)
+            self.metadata_db.mark_tenant_materialized(main_tenant_id)
 
     def _create_database_structure(
         self,
         database_name: str,
         branch_name: str = "main",
         description: Optional[str] = None,
+        create_tenant_files: bool = False,
     ) -> None:
         """Create the directory structure for a database.
 
@@ -119,8 +251,12 @@ class ProjectInitializer:
         tenant_dir = branch_path / "tenants"
         tenant_dir.mkdir(exist_ok=True)
 
-        # Create and initialize main tenant database
-        self._init_tenant_database(tenant_dir / "main.db")
+        # Create main tenant database in sharded directory (only if requested)
+        if create_tenant_files:
+            main_shard = _calculate_shard("main")
+            main_shard_dir = tenant_dir / main_shard
+            main_shard_dir.mkdir(parents=True, exist_ok=True)
+            self._init_tenant_database(main_shard_dir / "main.db")
 
     def _init_tenant_database(self, db_path: Path) -> None:
         """Initialize a tenant database with proper PRAGMAs.
@@ -140,6 +276,46 @@ class ProjectInitializer:
             # - foreign_keys = ON
             pass
 
+    def materialize_database(self, database_name: str) -> None:
+        """Materialize a lazy database into actual database structure.
+        
+        Args:
+            database_name: Name of the database to materialize
+            
+        Raises:
+            ValueError: If database doesn't exist or is already materialized
+        """
+        # Get database info from metadata
+        db_info = self.metadata_db.get_database(database_name)
+        if not db_info:
+            raise ValueError(f"Database '{database_name}' does not exist")
+            
+        # Check if already materialized
+        if db_info['materialized']:
+            return  # Already materialized
+            
+        db_path = self.config_dir / "databases" / database_name
+        if db_path.exists():
+            # Mark as materialized in metadata if directory already exists
+            self.metadata_db.mark_database_materialized(db_info['id'])
+            return
+            
+        # Get metadata details
+        metadata = json.loads(db_info['metadata']) if db_info['metadata'] else {}
+        branch_name = metadata.get("initial_branch", "main")
+        description = db_info.get('description')
+        
+        # Create the actual database structure (no tenant files - those are created when tables are added)
+        self._create_database_structure(database_name, branch_name, description, create_tenant_files=False)
+        
+        # Mark database as materialized in metadata
+        self.metadata_db.mark_database_materialized(db_info['id'])
+        
+        # Also mark the initial branch as materialized
+        branch_info = self.metadata_db.get_branch(db_info['id'], branch_name)
+        if branch_info:
+            self.metadata_db.mark_branch_materialized(branch_info['id'])
+
     def _save_config(self, config: ProjectConfig) -> None:
         """Save configuration to disk.
 
@@ -194,6 +370,7 @@ def init_database(
     database_name: str = "main",
     branch_name: str = "main",
     description: Optional[str] = None,
+    lazy: bool = True,
 ) -> None:
     """Initialize a new database within an existing project.
 
@@ -205,10 +382,11 @@ def init_database(
         database_name: Name for the database
         branch_name: Initial branch name (default: "main")
         description: Optional description
+        lazy: If True, don't create actual database files until first use
 
     Raises:
         FileNotFoundError: If project doesn't exist
         FileExistsError: If database already exists
     """
     initializer = ProjectInitializer(project_dir)
-    initializer.init_database(database_name, branch_name, description)
+    initializer.init_database(database_name, branch_name, description, lazy)

cinchdb/core/path_utils.py

@@ -1,7 +1,7 @@
 """Path utilities for CinchDB."""
 
 from pathlib import Path
-from typing import List
+from typing import List, Optional
 
 
 def get_project_root(start_path: Path) -> Path:
@@ -73,7 +73,7 @@ def get_tenant_path(
 def get_tenant_db_path(
     project_root: Path, database: str, branch: str, tenant: str
 ) -> Path:
-    """Get path to tenant database file.
+    """Get path to tenant database file using hash-based sharding.
 
     Args:
         project_root: Project root directory
@@ -82,9 +82,19 @@ def get_tenant_db_path(
         tenant: Tenant name
 
     Returns:
-        Path to tenant database file
+        Path to tenant database file in sharded directory structure
     """
-    return get_tenant_path(project_root, database, branch, tenant) / f"{tenant}.db"
+    import hashlib
+    
+    # Calculate shard using SHA256 hash (same as TenantManager)
+    hash_val = hashlib.sha256(tenant.encode('utf-8')).hexdigest()
+    shard = hash_val[:2]
+    
+    # Build sharded path: /tenants/{shard}/{tenant}.db
+    tenants_dir = get_tenant_path(project_root, database, branch, tenant)
+    shard_dir = tenants_dir / shard
+    
+    return shard_dir / f"{tenant}.db"
 
 
 def ensure_directory(path: Path) -> None:
@@ -105,11 +115,14 @@ def list_databases(project_root: Path) -> List[str]:
     Returns:
         List of database names
     """
-    db_dir = project_root / ".cinchdb" / "databases"
-    if not db_dir.exists():
+    metadata_db_path = project_root / ".cinchdb" / "metadata.db"
+    if not metadata_db_path.exists():
         return []
-
-    return sorted([d.name for d in db_dir.iterdir() if d.is_dir()])
+    
+    from cinchdb.infrastructure.metadata_db import MetadataDB
+    with MetadataDB(project_root) as metadata_db:
+        db_records = metadata_db.list_databases()
+        return sorted(record['name'] for record in db_records)
 
 
 def list_branches(project_root: Path, database: str) -> List[str]:
@@ -122,11 +135,17 @@ def list_branches(project_root: Path, database: str) -> List[str]:
     Returns:
         List of branch names
     """
-    branches_dir = get_database_path(project_root, database) / "branches"
-    if not branches_dir.exists():
+    metadata_db_path = project_root / ".cinchdb" / "metadata.db"
+    if not metadata_db_path.exists():
         return []
-
-    return sorted([b.name for b in branches_dir.iterdir() if b.is_dir()])
+    
+    from cinchdb.infrastructure.metadata_db import MetadataDB
+    with MetadataDB(project_root) as metadata_db:
+        db_info = metadata_db.get_database(database)
+        if not db_info:
+            return []
+        branch_records = metadata_db.list_branches(db_info['id'])
+        return sorted(record['name'] for record in branch_records)
 
 
 def list_tenants(project_root: Path, database: str, branch: str) -> List[str]:
@@ -140,19 +159,17 @@ def list_tenants(project_root: Path, database: str, branch: str) -> List[str]:
     Returns:
         List of tenant names
     """
-    tenants_dir = get_branch_path(project_root, database, branch) / "tenants"
-    if not tenants_dir.exists():
+    metadata_db_path = project_root / ".cinchdb" / "metadata.db"
+    if not metadata_db_path.exists():
         return []
-
-    # List both .db files and .meta files for lazy tenants
-    tenants = set()
-    for f in tenants_dir.iterdir():
-        if f.is_file():
-            if f.suffix == ".db":
-                tenants.add(f.stem)
-            elif f.suffix == ".meta" and f.name.startswith("."):
-                # Lazy tenant metadata files are named .{tenant_name}.meta
-                tenant_name = f.stem[1:]  # Remove leading dot
-                tenants.add(tenant_name)
-
-    return sorted(list(tenants))
+    
+    from cinchdb.infrastructure.metadata_db import MetadataDB
+    with MetadataDB(project_root) as metadata_db:
+        db_info = metadata_db.get_database(database)
+        if not db_info:
+            return []
+        branch_info = metadata_db.get_branch(db_info['id'], branch)
+        if not branch_info:
+            return []
+        tenant_records = metadata_db.list_tenants(branch_info['id'])
+        return sorted(record['name'] for record in tenant_records)

cinchdb/infrastructure/metadata_connection_pool.py

@@ -0,0 +1,145 @@
+"""Connection pool for MetadataDB to ensure efficient connection reuse."""
+
+import threading
+from pathlib import Path
+from typing import Optional, Dict
+from weakref import WeakValueDictionary
+
+from cinchdb.infrastructure.metadata_db import MetadataDB
+
+
+class MetadataConnectionPool:
+    """Thread-safe, lazy-initialized connection pool for MetadataDB.
+    
+    Uses a singleton pattern per project directory to ensure connection reuse
+    across all managers and operations within a project.
+    """
+    
+    _instances: Dict[str, 'MetadataConnectionPool'] = {}
+    _lock = threading.Lock()
+    
+    def __init__(self, project_path: Path):
+        """Initialize the connection pool (but don't create connection yet).
+        
+        Args:
+            project_path: Path to the project directory
+        """
+        self.project_path = Path(project_path)
+        self._connection: Optional[MetadataDB] = None
+        self._connection_lock = threading.Lock()
+        self._ref_count = 0
+        
+    @classmethod
+    def get_instance(cls, project_path: Path) -> 'MetadataConnectionPool':
+        """Get or create a connection pool for the given project.
+        
+        Args:
+            project_path: Path to the project directory
+            
+        Returns:
+            MetadataConnectionPool instance for this project
+        """
+        path_str = str(project_path.resolve())
+        
+        # Fast path - check if instance exists
+        if path_str in cls._instances:
+            return cls._instances[path_str]
+        
+        # Slow path - create new instance with lock
+        with cls._lock:
+            # Double-check pattern
+            if path_str not in cls._instances:
+                cls._instances[path_str] = cls(project_path)
+            return cls._instances[path_str]
+    
+    def get_connection(self) -> MetadataDB:
+        """Get or create a MetadataDB connection (lazy initialization).
+        
+        Returns:
+            MetadataDB instance (shared across all callers for this project)
+        """
+        # Fast path - connection already exists
+        if self._connection is not None:
+            return self._connection
+        
+        # Slow path - create connection with lock
+        with self._connection_lock:
+            # Double-check pattern
+            if self._connection is None:
+                self._connection = MetadataDB(self.project_path)
+            return self._connection
+    
+    def acquire(self) -> MetadataDB:
+        """Acquire a reference to the connection.
+        
+        Returns:
+            MetadataDB instance
+        """
+        with self._connection_lock:
+            self._ref_count += 1
+            return self.get_connection()
+    
+    def release(self) -> None:
+        """Release a reference to the connection.
+        
+        When ref count reaches 0, we could close the connection,
+        but we keep it open for performance since SQLite handles
+        concurrent access well with WAL mode.
+        """
+        with self._connection_lock:
+            self._ref_count = max(0, self._ref_count - 1)
+    
+    def close(self) -> None:
+        """Explicitly close the connection (called on shutdown)."""
+        with self._connection_lock:
+            if self._connection is not None:
+                self._connection.close()
+                self._connection = None
+                self._ref_count = 0
+    
+    @classmethod
+    def close_all(cls) -> None:
+        """Close all connection pools (useful for cleanup in tests)."""
+        with cls._lock:
+            for pool in cls._instances.values():
+                pool.close()
+            cls._instances.clear()
+
+
+class MetadataDBHandle:
+    """Context manager for safely acquiring and releasing metadata connections."""
+    
+    def __init__(self, project_path: Path):
+        """Initialize handle for metadata connection.
+        
+        Args:
+            project_path: Path to the project directory
+        """
+        self.pool = MetadataConnectionPool.get_instance(project_path)
+        self.connection: Optional[MetadataDB] = None
+    
+    def __enter__(self) -> MetadataDB:
+        """Acquire connection from pool."""
+        self.connection = self.pool.acquire()
+        return self.connection
+    
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        """Release connection back to pool."""
+        self.pool.release()
+        self.connection = None
+
+
+def get_metadata_db(project_path: Path) -> MetadataDB:
+    """Get a metadata database connection from the pool.
+    
+    This is a convenience function for code that doesn't use context managers.
+    The connection is shared and should NOT be closed by the caller.
+    
+    Args:
+        project_path: Path to the project directory
+        
+    Returns:
+        MetadataDB instance (shared, do not close)
+    """
+    pool = MetadataConnectionPool.get_instance(project_path)
+    return pool.get_connection()