spatial-memory-mcp 1.0.3__py3-none-any.whl → 1.6.0__py3-none-any.whl

spatial_memory/core/database.py

@@ -34,8 +34,21 @@ import pyarrow.parquet as pq
 from filelock import FileLock, Timeout as FileLockTimeout
 
 from spatial_memory.core.connection_pool import ConnectionPool
-from spatial_memory.core.errors import FileLockError, MemoryNotFoundError, StorageError, ValidationError
-from spatial_memory.core.utils import utc_now
+from spatial_memory.core.db_idempotency import IdempotencyManager, IdempotencyRecord
+from spatial_memory.core.db_indexes import IndexManager
+from spatial_memory.core.db_migrations import CURRENT_SCHEMA_VERSION, MigrationManager
+from spatial_memory.core.db_search import SearchManager
+from spatial_memory.core.db_versioning import VersionManager
+from spatial_memory.core.errors import (
+    DimensionMismatchError,
+    FileLockError,
+    MemoryNotFoundError,
+    PartialBatchInsertError,
+    StorageError,
+    ValidationError,
+)
+from spatial_memory.core.filesystem import detect_filesystem_type, get_filesystem_warning_message, is_network_filesystem
+from spatial_memory.core.utils import to_aware_utc, utc_now
 
 # Import centralized validation functions
 from spatial_memory.core.validation import (
@@ -131,9 +144,14 @@ def invalidate_connection(storage_path: Path) -> bool:
 # Retry Decorator
 # ============================================================================
 
+# Default retry settings (can be overridden per-call)
+DEFAULT_RETRY_MAX_ATTEMPTS = 3
+DEFAULT_RETRY_BACKOFF_SECONDS = 0.5
+
+
 def retry_on_storage_error(
-    max_attempts: int = 3,
-    backoff: float = 0.5,
+    max_attempts: int = DEFAULT_RETRY_MAX_ATTEMPTS,
+    backoff: float = DEFAULT_RETRY_BACKOFF_SECONDS,
 ) -> Callable[[F], F]:
     """Retry decorator for transient storage errors.
 
@@ -483,8 +501,8 @@ class Database:
         enable_fts: bool = True,
         index_nprobes: int = 20,
         index_refine_factor: int = 5,
-        max_retry_attempts: int = 3,
-        retry_backoff_seconds: float = 0.5,
+        max_retry_attempts: int = DEFAULT_RETRY_MAX_ATTEMPTS,
+        retry_backoff_seconds: float = DEFAULT_RETRY_BACKOFF_SECONDS,
         read_consistency_interval_ms: int = 0,
         index_wait_timeout_seconds: float = 30.0,
         fts_stem: bool = True,
@@ -498,6 +516,7 @@ class Database:
         filelock_enabled: bool = True,
         filelock_timeout: float = 30.0,
         filelock_poll_interval: float = 0.1,
+        acknowledge_network_filesystem_risk: bool = False,
     ) -> None:
         """Initialize the database connection.
 
@@ -524,6 +543,7 @@ class Database:
             hnsw_ef_construction: HNSW build-time search width (100-1000).
             enable_memory_expiration: Enable automatic memory expiration.
             default_memory_ttl_days: Default TTL for memories in days (None = no expiration).
+            acknowledge_network_filesystem_risk: Suppress network filesystem warnings.
         """
         self.storage_path = Path(storage_path)
         self.embedding_dim = embedding_dim
@@ -547,6 +567,7 @@ class Database:
         self.filelock_enabled = filelock_enabled
         self.filelock_timeout = filelock_timeout
         self.filelock_poll_interval = filelock_poll_interval
+        self.acknowledge_network_filesystem_risk = acknowledge_network_filesystem_risk
         self._db: lancedb.DBConnection | None = None
         self._table: LanceTable | None = None
         self._has_vector_index: bool | None = None
@@ -564,6 +585,18 @@ class Database:
         self._write_lock = threading.RLock()
         # Cross-process lock (initialized in connect())
         self._process_lock: ProcessLockManager | None = None
+        # Auto-compaction tracking
+        self._modification_count: int = 0
+        self._auto_compaction_threshold: int = 100  # Compact after this many modifications
+        self._auto_compaction_enabled: bool = True
+        # Version manager (initialized in connect())
+        self._version_manager: VersionManager | None = None
+        # Index manager (initialized in connect())
+        self._index_manager: IndexManager | None = None
+        # Search manager (initialized in connect())
+        self._search_manager: SearchManager | None = None
+        # Idempotency manager (initialized in connect())
+        self._idempotency_manager: IdempotencyManager | None = None
 
     def __enter__(self) -> Database:
         """Enter context manager."""
@@ -579,6 +612,13 @@ class Database:
         try:
             self.storage_path.mkdir(parents=True, exist_ok=True)
 
+            # Check for network filesystem and warn if detected
+            if not self.acknowledge_network_filesystem_risk:
+                if is_network_filesystem(self.storage_path):
+                    fs_type = detect_filesystem_type(self.storage_path)
+                    warning_msg = get_filesystem_warning_message(fs_type, self.storage_path)
+                    logger.warning(warning_msg)
+
             # Initialize cross-process lock manager
             if self.filelock_enabled:
                 lock_path = self.storage_path / ".spatial-memory.lock"
@@ -597,109 +637,144 @@ class Database:
                 read_consistency_interval_ms=self.read_consistency_interval_ms,
             )
             self._ensure_table()
+            # Initialize remaining managers (IndexManager already initialized in _ensure_table)
+            self._version_manager = VersionManager(self)
+            self._search_manager = SearchManager(self)
+            self._idempotency_manager = IdempotencyManager(self)
             logger.info(f"Connected to LanceDB at {self.storage_path}")
+
+            # Check for pending schema migrations
+            self._check_pending_migrations()
         except Exception as e:
             raise StorageError(f"Failed to connect to database: {e}") from e
 
+    def _check_pending_migrations(self) -> None:
+        """Check for pending migrations and warn if any exist.
+
+        This method checks the schema version and logs a warning if there
+        are pending migrations. It does not auto-apply migrations - that
+        requires explicit user action via the CLI.
+        """
+        try:
+            manager = MigrationManager(self, embeddings=None)
+            manager.register_builtin_migrations()
+
+            current_version = manager.get_current_version()
+            pending = manager.get_pending_migrations()
+
+            if pending:
+                pending_versions = [m.version for m in pending]
+                logger.warning(
+                    f"Database schema version {current_version} is outdated. "
+                    f"{len(pending)} migration(s) pending: {', '.join(pending_versions)}. "
+                    f"Target version: {CURRENT_SCHEMA_VERSION}. "
+                    f"Run 'spatial-memory migrate' to apply migrations."
+                )
+        except Exception as e:
+            # Don't fail connection due to migration check errors
+            logger.debug(f"Migration check skipped: {e}")
+
     def _ensure_table(self) -> None:
-        """Ensure the memories table exists with appropriate indexes."""
+        """Ensure the memories table exists with appropriate indexes.
+
+        Uses retry logic to handle race conditions when multiple processes
+        attempt to create/open the table simultaneously.
+        """
         if self._db is None:
             raise StorageError("Database not connected")
 
-        existing_tables_result = self._db.list_tables()
-        # Handle both old (list) and new (object with .tables) LanceDB API
-        if hasattr(existing_tables_result, 'tables'):
-            existing_tables = existing_tables_result.tables
-        else:
-            existing_tables = existing_tables_result
-        if "memories" not in existing_tables:
-            # Create table with schema
-            schema = pa.schema([
-                pa.field("id", pa.string()),
-                pa.field("content", pa.string()),
-                pa.field("vector", pa.list_(pa.float32(), self.embedding_dim)),
-                pa.field("created_at", pa.timestamp("us")),
-                pa.field("updated_at", pa.timestamp("us")),
-                pa.field("last_accessed", pa.timestamp("us")),
-                pa.field("access_count", pa.int32()),
-                pa.field("importance", pa.float32()),
-                pa.field("namespace", pa.string()),
-                pa.field("tags", pa.list_(pa.string())),
-                pa.field("source", pa.string()),
-                pa.field("metadata", pa.string()),
-                pa.field("expires_at", pa.timestamp("us")),  # TTL support - nullable
-            ])
-            self._table = self._db.create_table("memories", schema=schema)
-            logger.info("Created memories table")
-
-            # Create FTS index on new table if enabled
-            if self.enable_fts:
-                self._create_fts_index()
-        else:
-            self._table = self._db.open_table("memories")
-            logger.debug("Opened existing memories table")
+        max_retries = 3
+        retry_delay = 0.1  # Start with 100ms
 
-            # Check existing indexes
-            self._check_existing_indexes()
+        for attempt in range(max_retries):
+            try:
+                existing_tables_result = self._db.list_tables()
+                # Handle both old (list) and new (object with .tables) LanceDB API
+                if hasattr(existing_tables_result, 'tables'):
+                    existing_tables = existing_tables_result.tables
+                else:
+                    existing_tables = existing_tables_result
+
+                if "memories" not in existing_tables:
+                    # Create table with schema
+                    schema = pa.schema([
+                        pa.field("id", pa.string()),
+                        pa.field("content", pa.string()),
+                        pa.field("vector", pa.list_(pa.float32(), self.embedding_dim)),
+                        pa.field("created_at", pa.timestamp("us")),
+                        pa.field("updated_at", pa.timestamp("us")),
+                        pa.field("last_accessed", pa.timestamp("us")),
+                        pa.field("access_count", pa.int32()),
+                        pa.field("importance", pa.float32()),
+                        pa.field("namespace", pa.string()),
+                        pa.field("tags", pa.list_(pa.string())),
+                        pa.field("source", pa.string()),
+                        pa.field("metadata", pa.string()),
+                        pa.field("expires_at", pa.timestamp("us")),  # TTL support - nullable
+                    ])
+                    try:
+                        self._table = self._db.create_table("memories", schema=schema)
+                        logger.info("Created memories table")
+                    except Exception as create_err:
+                        # Table might have been created by another process
+                        if "already exists" in str(create_err).lower():
+                            logger.debug("Table created by another process, opening it")
+                            self._table = self._db.open_table("memories")
+                        else:
+                            raise
 
-    def _check_existing_indexes(self) -> None:
-        """Check which indexes already exist using robust detection."""
-        try:
-            indices = self.table.list_indices()
+                    # Initialize IndexManager immediately after table is set
+                    self._index_manager = IndexManager(self)
 
-            self._has_vector_index = False
-            self._has_fts_index = False
+                    # Create FTS index on new table if enabled
+                    if self.enable_fts:
+                        self._index_manager.create_fts_index()
+                else:
+                    self._table = self._db.open_table("memories")
+                    logger.debug("Opened existing memories table")
 
-            for idx in indices:
-                index_name = str(_get_index_attr(idx, "name", "")).lower()
-                index_type = str(_get_index_attr(idx, "index_type", "")).upper()
-                columns = _get_index_attr(idx, "columns", [])
+                    # Initialize IndexManager immediately after table is set
+                    self._index_manager = IndexManager(self)
 
-                # Vector index detection: check index_type or column name
-                if index_type in VECTOR_INDEX_TYPES:
-                    self._has_vector_index = True
-                elif "vector" in columns or "vector" in index_name:
-                    self._has_vector_index = True
+                    # Check existing indexes
+                    self._index_manager.check_existing_indexes()
 
-                # FTS index detection: check index_type or name patterns
-                if index_type == "FTS":
-                    self._has_fts_index = True
-                elif "fts" in index_name or "content" in index_name:
-                    self._has_fts_index = True
+                # Success - exit retry loop
+                return
 
-            logger.debug(
-                f"Existing indexes: vector={self._has_vector_index}, "
-                f"fts={self._has_fts_index}"
-            )
-        except Exception as e:
-            logger.warning(f"Could not check existing indexes: {e}")
-            self._has_vector_index = None
-            self._has_fts_index = None
+            except Exception as e:
+                error_msg = str(e).lower()
+                # Retry on transient race conditions
+                if attempt < max_retries - 1 and (
+                    "not found" in error_msg
+                    or "does not exist" in error_msg
+                    or "already exists" in error_msg
+                ):
+                    logger.debug(
+                        f"Table operation failed (attempt {attempt + 1}/{max_retries}), "
+                        f"retrying in {retry_delay}s: {e}"
+                    )
+                    time.sleep(retry_delay)
+                    retry_delay *= 2  # Exponential backoff
+                else:
+                    raise
+
+    def _check_existing_indexes(self) -> None:
+        """Check which indexes already exist. Delegates to IndexManager."""
+        if self._index_manager is None:
+            raise StorageError("Database not connected")
+        self._index_manager.check_existing_indexes()
+        # Sync local state for backward compatibility
+        self._has_vector_index = self._index_manager.has_vector_index
+        self._has_fts_index = self._index_manager.has_fts_index
 
     def _create_fts_index(self) -> None:
-        """Create full-text search index with optimized settings."""
-        try:
-            self.table.create_fts_index(
-                "content",
-                use_tantivy=False,  # Use Lance native FTS
-                language=self.fts_language,
-                stem=self.fts_stem,
-                remove_stop_words=self.fts_remove_stop_words,
-                with_position=True,  # Enable phrase queries
-                lower_case=True,  # Case-insensitive search
-            )
-            self._has_fts_index = True
-            logger.info(
-                f"Created FTS index with stemming={self.fts_stem}, "
-                f"stop_words={self.fts_remove_stop_words}"
-            )
-        except Exception as e:
-            # Check if index already exists (not an error)
-            if "already exists" in str(e).lower():
-                self._has_fts_index = True
-                logger.debug("FTS index already exists")
-            else:
-                logger.warning(f"FTS index creation failed: {e}")
+        """Create FTS index. Delegates to IndexManager."""
+        if self._index_manager is None:
+            raise StorageError("Database not connected")
+        self._index_manager.create_fts_index()
+        # Sync local state for backward compatibility
+        self._has_fts_index = self._index_manager.has_fts_index
 
     @property
     def table(self) -> LanceTable:
@@ -710,18 +785,30 @@ class Database:
         return self._table
 
     def close(self) -> None:
-        """Close the database connection (connection remains pooled)."""
+        """Close the database connection and remove from pool.
+
+        This invalidates the pooled connection so that subsequent
+        Database instances will create fresh connections.
+        """
+        # Invalidate pooled connection first
+        invalidate_connection(self.storage_path)
+
+        # Clear local state
         self._table = None
         self._db = None
         self._has_vector_index = None
         self._has_fts_index = None
+        self._version_manager = None
+        self._index_manager = None
+        self._search_manager = None
+        self._idempotency_manager = None
         with self._cache_lock:
             self._cached_row_count = None
             self._count_cache_time = 0.0
         with self._namespace_cache_lock:
             self._cached_namespaces = None
             self._namespace_cache_time = 0.0
-        logger.debug("Database connection closed")
+        logger.debug("Database connection closed and removed from pool")
 
     def reconnect(self) -> None:
         """Invalidate cached connection and reconnect.
@@ -781,313 +868,86 @@ class Database:
             self._cached_namespaces = None
             self._namespace_cache_time = 0.0
 
-    # ========================================================================
-    # Index Management
-    # ========================================================================
-
-    def create_vector_index(self, force: bool = False) -> bool:
-        """Create vector index for similarity search.
-
-        Supports IVF_PQ, IVF_FLAT, and HNSW_SQ index types based on configuration.
-        Automatically determines optimal parameters based on dataset size.
+    def _track_modification(self, count: int = 1) -> None:
+        """Track database modifications and trigger auto-compaction if threshold reached.
 
         Args:
-            force: Force index creation regardless of dataset size.
-
-        Returns:
-            True if index was created, False if skipped.
-
-        Raises:
-            StorageError: If index creation fails.
+            count: Number of modifications to track (default 1).
         """
-        count = self.table.count_rows()
-
-        # Check threshold
-        if count < self.vector_index_threshold and not force:
-            logger.info(
-                f"Dataset has {count} rows, below threshold {self.vector_index_threshold}. "
-                "Skipping vector index creation."
-            )
-            return False
-
-        # Check if already exists
-        if self._has_vector_index and not force:
-            logger.info("Vector index already exists")
-            return False
-
-        # Handle HNSW_SQ index type
-        if self.index_type == "HNSW_SQ":
-            return self._create_hnsw_index(count)
-
-        # IVF-based index creation (IVF_PQ or IVF_FLAT)
-        return self._create_ivf_index(count)
+        if not self._auto_compaction_enabled:
+            return
 
-    def _create_hnsw_index(self, count: int) -> bool:
-        """Create HNSW-SQ vector index.
+        self._modification_count += count
+        if self._modification_count >= self._auto_compaction_threshold:
+            # Reset counter before compacting to avoid re-triggering
+            self._modification_count = 0
+            try:
+                stats = self._get_table_stats()
+                # Only compact if there are enough fragments to justify it
+                if stats.get("num_small_fragments", 0) >= 5:
+                    logger.info(
+                        f"Auto-compaction triggered after {self._auto_compaction_threshold} "
+                        f"modifications ({stats.get('num_small_fragments', 0)} small fragments)"
+                    )
+                    self.table.compact_files()
+                    logger.debug("Auto-compaction completed")
+            except Exception as e:
+                # Don't fail operations due to compaction issues
+                logger.debug(f"Auto-compaction skipped: {e}")
 
-        HNSW (Hierarchical Navigable Small World) provides better recall than IVF
-        at the cost of higher memory usage. Good for datasets where recall is critical.
+    def set_auto_compaction(
+        self,
+        enabled: bool = True,
+        threshold: int | None = None,
+    ) -> None:
+        """Configure auto-compaction behavior.
 
         Args:
-            count: Number of rows in the table.
-
-        Returns:
-            True if index was created.
-
-        Raises:
-            StorageError: If index creation fails.
+            enabled: Whether auto-compaction is enabled.
+            threshold: Number of modifications before auto-compact (default: 100).
         """
-        logger.info(
-            f"Creating HNSW_SQ vector index: m={self.hnsw_m}, "
-            f"ef_construction={self.hnsw_ef_construction} for {count} rows"
-        )
+        self._auto_compaction_enabled = enabled
+        if threshold is not None:
+            if threshold < 10:
+                raise ValueError("Auto-compaction threshold must be at least 10")
+            self._auto_compaction_threshold = threshold
 
-        try:
-            self.table.create_index(
-                metric="cosine",
-                vector_column_name="vector",
-                index_type="HNSW_SQ",
-                replace=True,
-                m=self.hnsw_m,
-                ef_construction=self.hnsw_ef_construction,
-            )
-
-            # Wait for index to be ready with configurable timeout
-            self._wait_for_index_ready("vector", self.index_wait_timeout_seconds)
-
-            self._has_vector_index = True
-            logger.info("HNSW_SQ vector index created successfully")
-
-            # Optimize after index creation (may fail in some environments)
-            try:
-                self.table.optimize()
-            except Exception as optimize_error:
-                logger.debug(f"Optimization after index creation skipped: {optimize_error}")
-
-            return True
-
-        except Exception as e:
-            logger.error(f"Failed to create HNSW_SQ vector index: {e}")
-            raise StorageError(f"HNSW_SQ vector index creation failed: {e}") from e
-
-    def _create_ivf_index(self, count: int) -> bool:
-        """Create IVF-PQ or IVF-FLAT vector index.
+    # ========================================================================
+    # Index Management (delegates to IndexManager)
+    # ========================================================================
 
-        Uses sqrt rule for partitions: num_partitions = sqrt(count), clamped to [16, 4096].
-        Uses 48 sub-vectors for <500K rows (8 dims each for 384-dim vectors),
-        96 sub-vectors for >=500K rows (4 dims each).
+    def create_vector_index(self, force: bool = False) -> bool:
+        """Create vector index for similarity search. Delegates to IndexManager.
 
         Args:
-            count: Number of rows in the table.
+            force: Force index creation regardless of dataset size.
 
         Returns:
-            True if index was created.
+            True if index was created, False if skipped.
 
         Raises:
             StorageError: If index creation fails.
         """
-        # Use sqrt rule for partitions, clamped to [16, 4096]
-        num_partitions = int(math.sqrt(count))
-        num_partitions = max(16, min(num_partitions, 4096))
-
-        # Choose num_sub_vectors based on dataset size
-        # <500K: 48 sub-vectors (8 dims each for 384-dim, more precision)
-        # >=500K: 96 sub-vectors (4 dims each, more compression)
-        if count < 500_000:
-            num_sub_vectors = 48
-        else:
-            num_sub_vectors = 96
-
-        # Validate embedding_dim % num_sub_vectors == 0 (required for IVF-PQ)
-        if self.embedding_dim % num_sub_vectors != 0:
-            # Find a valid divisor from common sub-vector counts
-            valid_divisors = [96, 48, 32, 24, 16, 12, 8, 4]
-            found_divisor = False
-            for divisor in valid_divisors:
-                if self.embedding_dim % divisor == 0:
-                    logger.info(
-                        f"Adjusted num_sub_vectors from {num_sub_vectors} to {divisor} "
-                        f"for embedding_dim={self.embedding_dim}"
-                    )
-                    num_sub_vectors = divisor
-                    found_divisor = True
-                    break
-
-            if not found_divisor:
-                raise StorageError(
-                    f"Cannot create IVF-PQ index: embedding_dim={self.embedding_dim} "
-                    "has no suitable divisor for sub-vectors. "
-                    f"Tried divisors: {valid_divisors}"
-                )
-
-        # IVF-PQ requires minimum rows for training (sample_rate * num_partitions / 256)
-        # Default sample_rate=256, so we need at least 256 rows
-        # Also, IVF requires num_partitions < num_vectors for KMeans training
-        sample_rate = 256  # default
-        if count < 256:
-            # Use IVF_FLAT for very small datasets (no PQ training required)
-            logger.info(
-                f"Dataset too small for IVF-PQ ({count} rows < 256). "
-                "Using IVF_FLAT index instead."
-            )
-            index_type = "IVF_FLAT"
-            sample_rate = max(16, count // 4)  # Lower sample rate for small data
-        else:
-            index_type = self.index_type if self.index_type in ("IVF_PQ", "IVF_FLAT") else "IVF_PQ"
-
-        # Ensure num_partitions < num_vectors for KMeans clustering
-        if num_partitions >= count:
-            num_partitions = max(1, count // 4)  # Use 1/4 of count, minimum 1
-            logger.info(f"Adjusted num_partitions to {num_partitions} for {count} rows")
-
-        logger.info(
-            f"Creating {index_type} vector index: {num_partitions} partitions, "
-            f"{num_sub_vectors} sub-vectors for {count} rows"
-        )
-
-        try:
-            # LanceDB 0.27+ API: parameters passed directly to create_index
-            index_kwargs: dict[str, Any] = {
-                "metric": "cosine",
-                "num_partitions": num_partitions,
-                "vector_column_name": "vector",
-                "index_type": index_type,
-                "replace": True,
-                "sample_rate": sample_rate,
-            }
-
-            # num_sub_vectors only applies to PQ-based indexes
-            if "PQ" in index_type:
-                index_kwargs["num_sub_vectors"] = num_sub_vectors
-
-            self.table.create_index(**index_kwargs)
-
-            # Wait for index to be ready with configurable timeout
-            self._wait_for_index_ready("vector", self.index_wait_timeout_seconds)
-
-            self._has_vector_index = True
-            logger.info(f"{index_type} vector index created successfully")
-
-            # Optimize after index creation (may fail in some environments)
-            try:
-                self.table.optimize()
-            except Exception as optimize_error:
-                logger.debug(f"Optimization after index creation skipped: {optimize_error}")
-
-            return True
-
-        except Exception as e:
-            logger.error(f"Failed to create {index_type} vector index: {e}")
-            raise StorageError(f"{index_type} vector index creation failed: {e}") from e
-
-    def _wait_for_index_ready(
-        self,
-        column_name: str,
-        timeout_seconds: float,
-        poll_interval: float = 0.5,
-    ) -> None:
-        """Wait for an index on the specified column to be ready.
-
-        Args:
-            column_name: Name of the column the index is on (e.g., "vector").
-                         LanceDB typically names indexes as "{column_name}_idx".
-            timeout_seconds: Maximum time to wait.
-            poll_interval: Time between status checks.
-        """
-        if timeout_seconds <= 0:
-            return
-
-        start_time = time.time()
-        while time.time() - start_time < timeout_seconds:
-            try:
-                indices = self.table.list_indices()
-                for idx in indices:
-                    idx_name = str(_get_index_attr(idx, "name", "")).lower()
-                    idx_columns = _get_index_attr(idx, "columns", [])
-
-                    # Match by column name in index metadata, or index name contains column
-                    if column_name in idx_columns or column_name in idx_name:
-                        # Index exists, check if it's ready
-                        status = str(_get_index_attr(idx, "status", "ready"))
-                        if status.lower() in ("ready", "complete", "built"):
-                            logger.debug(f"Index on {column_name} is ready")
-                            return
-                        break
-            except Exception as e:
-                logger.debug(f"Error checking index status: {e}")
-
-            time.sleep(poll_interval)
-
-        logger.warning(
-            f"Timeout waiting for index on {column_name} after {timeout_seconds}s"
-        )
+        if self._index_manager is None:
+            raise StorageError("Database not connected")
+        result = self._index_manager.create_vector_index(force=force)
+        # Sync local state only when index was created or modified
+        if result:
+            self._has_vector_index = self._index_manager.has_vector_index
+        return result
 
     def create_scalar_indexes(self) -> None:
-        """Create scalar indexes for frequently filtered columns.
-
-        Creates:
-        - BTREE on id (fast lookups, upserts)
-        - BTREE on timestamps and importance (range queries)
-        - BITMAP on namespace and source (low cardinality)
-        - LABEL_LIST on tags (array contains queries)
+        """Create scalar indexes for frequently filtered columns. Delegates to IndexManager.
 
         Raises:
             StorageError: If index creation fails critically.
         """
-        # BTREE indexes for range queries and lookups
-        btree_columns = [
-            "id",  # Fast lookups and merge_insert
-            "created_at",
-            "updated_at",
-            "last_accessed",
-            "importance",
-            "access_count",
-            "expires_at",  # TTL expiration queries
-        ]
-
-        for column in btree_columns:
-            try:
-                self.table.create_scalar_index(
-                    column,
-                    index_type="BTREE",
-                    replace=True,
-                )
-                logger.debug(f"Created BTREE index on {column}")
-            except Exception as e:
-                if "already exists" not in str(e).lower():
-                    logger.warning(f"Could not create BTREE index on {column}: {e}")
-
-        # BITMAP indexes for low-cardinality columns
-        bitmap_columns = ["namespace", "source"]
-
-        for column in bitmap_columns:
-            try:
-                self.table.create_scalar_index(
-                    column,
-                    index_type="BITMAP",
-                    replace=True,
-                )
-                logger.debug(f"Created BITMAP index on {column}")
-            except Exception as e:
-                if "already exists" not in str(e).lower():
-                    logger.warning(f"Could not create BITMAP index on {column}: {e}")
-
-        # LABEL_LIST index for tags array (supports array_has_any queries)
-        try:
-            self.table.create_scalar_index(
-                "tags",
-                index_type="LABEL_LIST",
-                replace=True,
-            )
-            logger.debug("Created LABEL_LIST index on tags")
-        except Exception as e:
-            if "already exists" not in str(e).lower():
-                logger.warning(f"Could not create LABEL_LIST index on tags: {e}")
-
-        logger.info("Scalar indexes created")
+        if self._index_manager is None:
+            raise StorageError("Database not connected")
+        self._index_manager.create_scalar_indexes()
 
     def ensure_indexes(self, force: bool = False) -> dict[str, bool]:
-        """Ensure all appropriate indexes exist.
+        """Ensure all appropriate indexes exist. Delegates to IndexManager.
 
         Args:
             force: Force index creation regardless of thresholds.
@@ -1095,35 +955,12 @@ class Database:
         Returns:
             Dict indicating which indexes were created.
         """
-        results = {
-            "vector_index": False,
-            "scalar_indexes": False,
-            "fts_index": False,
-        }
-
-        count = self.table.count_rows()
-
-        # Vector index
-        if self.auto_create_indexes or force:
-            if count >= self.vector_index_threshold or force:
-                results["vector_index"] = self.create_vector_index(force=force)
-
-        # Scalar indexes (always create if > 1000 rows)
-        if count >= 1000 or force:
-            try:
-                self.create_scalar_indexes()
-                results["scalar_indexes"] = True
-            except Exception as e:
-                logger.warning(f"Scalar index creation partially failed: {e}")
-
-        # FTS index
-        if self.enable_fts and not self._has_fts_index:
-            try:
-                self._create_fts_index()
-                results["fts_index"] = True
-            except Exception as e:
-                logger.warning(f"FTS index creation failed in ensure_indexes: {e}")
-
+        if self._index_manager is None:
+            raise StorageError("Database not connected")
+        results = self._index_manager.ensure_indexes(force=force)
+        # Sync local state for backward compatibility
+        self._has_vector_index = self._index_manager.has_vector_index
+        self._has_fts_index = self._index_manager.has_fts_index
         return results
 
     # ========================================================================
@@ -1292,6 +1129,13 @@ class Database:
         if not 0.0 <= importance <= 1.0:
             raise ValidationError("Importance must be between 0.0 and 1.0")
 
+        # Validate vector dimensions
+        if len(vector) != self.embedding_dim:
+            raise DimensionMismatchError(
+                expected_dim=self.embedding_dim,
+                actual_dim=len(vector),
+            )
+
         memory_id = str(uuid.uuid4())
         now = utc_now()
 
@@ -1319,6 +1163,7 @@ class Database:
         try:
             self.table.add([record])
             self._invalidate_count_cache()
+            self._track_modification()
             self._invalidate_namespace_cache()
             logger.debug(f"Inserted memory {memory_id}")
             return memory_id
@@ -1335,23 +1180,26 @@ class Database:
         self,
         records: list[dict[str, Any]],
         batch_size: int = 1000,
+        atomic: bool = False,
     ) -> list[str]:
         """Insert multiple memories efficiently with batching.
 
-        Note: Batch insert is NOT atomic. Partial failures may leave some
-        records inserted. If atomicity is required, use individual inserts
-        with transaction management at the application layer.
-
         Args:
             records: List of memory records with content, vector, and optional fields.
             batch_size: Records per batch (default: 1000, max: 10000).
+            atomic: If True, rollback all inserts on partial failure.
+                When atomic=True and a batch fails:
+                - Attempts to delete already-inserted records
+                - If rollback succeeds, raises the original StorageError
+                - If rollback fails, raises PartialBatchInsertError with succeeded_ids
 
         Returns:
             List of generated memory IDs.
 
         Raises:
             ValidationError: If input validation fails or batch_size exceeds maximum.
-            StorageError: If database operation fails.
+            StorageError: If database operation fails (and rollback succeeds when atomic=True).
+            PartialBatchInsertError: If atomic=True and rollback fails after partial insert.
         """
         if batch_size > self.MAX_BATCH_SIZE:
             raise ValidationError(
@@ -1359,9 +1207,10 @@ class Database:
             )
 
         all_ids: list[str] = []
+        total_requested = len(records)
 
         # Process in batches for large inserts
-        for i in range(0, len(records), batch_size):
+        for batch_index, i in enumerate(range(0, len(records), batch_size)):
             batch = records[i:i + batch_size]
             now = utc_now()
             memory_ids: list[str] = []
@@ -1389,6 +1238,14 @@ class Database:
                 else:
                     vector_list = raw_vector
 
+                # Validate vector dimensions
+                if len(vector_list) != self.embedding_dim:
+                    raise DimensionMismatchError(
+                        expected_dim=self.embedding_dim,
+                        actual_dim=len(vector_list),
+                        record_index=i + len(memory_ids),
+                    )
+
                 # Calculate expires_at if default TTL is configured
                 expires_at = None
                 if self.default_memory_ttl_days is not None:
@@ -1415,9 +1272,29 @@ class Database:
                 self.table.add(prepared_records)
                 all_ids.extend(memory_ids)
                 self._invalidate_count_cache()
+                self._track_modification(len(memory_ids))
                 self._invalidate_namespace_cache()
-                logger.debug(f"Inserted batch {i // batch_size + 1}: {len(memory_ids)} memories")
+                logger.debug(f"Inserted batch {batch_index + 1}: {len(memory_ids)} memories")
             except Exception as e:
+                if atomic and all_ids:
+                    # Attempt rollback of previously inserted records
+                    logger.warning(
+                        f"Batch {batch_index + 1} failed, attempting rollback of "
+                        f"{len(all_ids)} previously inserted records"
+                    )
+                    rollback_error = self._rollback_batch_insert(all_ids)
+                    if rollback_error:
+                        # Rollback failed - raise PartialBatchInsertError
+                        raise PartialBatchInsertError(
+                            message=f"Batch insert failed and rollback also failed: {e}",
+                            succeeded_ids=all_ids,
+                            total_requested=total_requested,
+                            failed_batch_index=batch_index,
+                        ) from e
+                    else:
+                        # Rollback succeeded - raise original error
+                        logger.info(f"Rollback successful, deleted {len(all_ids)} records")
+                        raise StorageError(f"Failed to insert batch (rolled back): {e}") from e
                 raise StorageError(f"Failed to insert batch: {e}") from e
 
         # Check if we should create indexes after large insert
@@ -1433,6 +1310,31 @@ class Database:
         logger.debug(f"Inserted {len(all_ids)} memories total")
         return all_ids
 
+    def _rollback_batch_insert(self, memory_ids: list[str]) -> Exception | None:
+        """Attempt to delete records inserted during a failed batch operation.
+
+        Args:
+            memory_ids: List of memory IDs to delete.
+
+        Returns:
+            None if rollback succeeded, Exception if it failed.
+        """
+        try:
+            if not memory_ids:
+                return None
+
+            # Use delete_batch for efficient rollback
+            id_list = ", ".join(f"'{_sanitize_string(mid)}'" for mid in memory_ids)
+            self.table.delete(f"id IN ({id_list})")
+            self._invalidate_count_cache()
+            self._track_modification(len(memory_ids))
+            self._invalidate_namespace_cache()
+            logger.debug(f"Rolled back {len(memory_ids)} records")
+            return None
+        except Exception as e:
+            logger.error(f"Rollback failed: {e}")
+            return e
+
     @with_stale_connection_recovery
     def get(self, memory_id: str) -> dict[str, Any]:
         """Get a memory by ID.
@@ -1467,6 +1369,51 @@ class Database:
         except Exception as e:
             raise StorageError(f"Failed to get memory: {e}") from e
 
+    def get_batch(self, memory_ids: list[str]) -> dict[str, dict[str, Any]]:
+        """Get multiple memories by ID in a single query.
+
+        Args:
+            memory_ids: List of memory UUIDs to retrieve.
+
+        Returns:
+            Dict mapping memory_id to memory record. Missing IDs are not included.
+
+        Raises:
+            ValidationError: If any memory_id format is invalid.
+            StorageError: If database operation fails.
+        """
+        if not memory_ids:
+            return {}
+
+        # Validate all IDs first
+        validated_ids: list[str] = []
+        for memory_id in memory_ids:
+            try:
+                validated_id = _validate_uuid(memory_id)
+                validated_ids.append(_sanitize_string(validated_id))
+            except Exception as e:
+                logger.debug(f"Invalid memory ID {memory_id}: {e}")
+                continue
+
+        if not validated_ids:
+            return {}
+
+        try:
+            # Batch fetch with single IN query
+            id_list = ", ".join(f"'{mid}'" for mid in validated_ids)
+            results = self.table.search().where(f"id IN ({id_list})").to_list()
+
+            # Build result map
+            result_map: dict[str, dict[str, Any]] = {}
+            for record in results:
+                # Deserialize metadata
+                record["metadata"] = json.loads(record["metadata"]) if record["metadata"] else {}
+                result_map[record["id"]] = record
+
+            return result_map
+        except Exception as e:
+            raise StorageError(f"Failed to batch get memories: {e}") from e
+
     @with_process_lock
     @with_write_lock
     def update(self, memory_id: str, updates: dict[str, Any]) -> None:
@@ -1522,6 +1469,108 @@ class Database:
         except Exception as e:
             raise StorageError(f"Failed to update memory: {e}") from e
 
+    @with_process_lock
+    @with_write_lock
+    def update_batch(
+        self, updates: list[tuple[str, dict[str, Any]]]
+    ) -> tuple[int, list[str]]:
+        """Update multiple memories using atomic merge_insert.
+
+        Args:
+            updates: List of (memory_id, updates_dict) tuples.
+
+        Returns:
+            Tuple of (success_count, list of failed memory_ids).
+
+        Raises:
+            StorageError: If database operation fails completely.
+        """
+        if not updates:
+            return 0, []
+
+        now = utc_now()
+        records_to_update: list[dict[str, Any]] = []
+        failed_ids: list[str] = []
+
+        # Validate all IDs and collect them
+        validated_updates: list[tuple[str, dict[str, Any]]] = []
+        for memory_id, update_dict in updates:
+            try:
+                validated_id = _validate_uuid(memory_id)
+                validated_updates.append((_sanitize_string(validated_id), update_dict))
+            except Exception as e:
+                logger.debug(f"Invalid memory ID {memory_id}: {e}")
+                failed_ids.append(memory_id)
+
+        if not validated_updates:
+            return 0, failed_ids
+
+        # Batch fetch all records
+        validated_ids = [vid for vid, _ in validated_updates]
+        try:
+            id_list = ", ".join(f"'{mid}'" for mid in validated_ids)
+            all_records = self.table.search().where(f"id IN ({id_list})").to_list()
+        except Exception as e:
+            logger.error(f"Failed to batch fetch records for update: {e}")
+            raise StorageError(f"Failed to batch fetch for update: {e}") from e
+
+        # Build lookup map
+        record_map: dict[str, dict[str, Any]] = {}
+        for record in all_records:
+            record_map[record["id"]] = record
+
+        # Apply updates to found records
+        update_dict_map = dict(validated_updates)
+        for memory_id in validated_ids:
+            if memory_id not in record_map:
+                logger.debug(f"Memory {memory_id} not found for batch update")
+                failed_ids.append(memory_id)
+                continue
+
+            record = record_map[memory_id]
+            update_dict = update_dict_map[memory_id]
+
+            # Apply updates
+            record["updated_at"] = now
+            for key, value in update_dict.items():
+                if key == "metadata" and isinstance(value, dict):
+                    record[key] = json.dumps(value)
+                elif key == "vector" and isinstance(value, np.ndarray):
+                    record[key] = value.tolist()
+                else:
+                    record[key] = value
+
+            # Ensure metadata is serialized
+            if isinstance(record.get("metadata"), dict):
+                record["metadata"] = json.dumps(record["metadata"])
+
+            # Ensure vector is a list
+            if isinstance(record.get("vector"), np.ndarray):
+                record["vector"] = record["vector"].tolist()
+
+            records_to_update.append(record)
+
+        if not records_to_update:
+            return 0, failed_ids
+
+        try:
+            # Atomic batch upsert
+            (
+                self.table.merge_insert("id")
+                .when_matched_update_all()
+                .when_not_matched_insert_all()
+                .execute(records_to_update)
+            )
+            success_count = len(records_to_update)
+            logger.debug(
+                f"Batch updated {success_count}/{len(updates)} memories "
+                "(atomic merge_insert)"
+            )
+            return success_count, failed_ids
+        except Exception as e:
+            logger.error(f"Failed to batch update: {e}")
+            raise StorageError(f"Failed to batch update: {e}") from e
+
     @with_process_lock
     @with_write_lock
     def delete(self, memory_id: str) -> None:
@@ -1545,6 +1594,7 @@ class Database:
         try:
             self.table.delete(f"id = '{safe_id}'")
             self._invalidate_count_cache()
+            self._track_modification()
             self._invalidate_namespace_cache()
             logger.debug(f"Deleted memory {memory_id}")
         except Exception as e:
@@ -1572,6 +1622,7 @@ class Database:
             count_before: int = self.table.count_rows()
             self.table.delete(f"namespace = '{safe_ns}'")
             self._invalidate_count_cache()
+            self._track_modification()
             self._invalidate_namespace_cache()
             count_after: int = self.table.count_rows()
             deleted = count_before - count_after
@@ -1615,6 +1666,7 @@ class Database:
                     self.table.delete("id IS NOT NULL")
 
             self._invalidate_count_cache()
+            self._track_modification()
             self._invalidate_namespace_cache()
 
             # Reset index tracking flags for test isolation
@@ -1634,6 +1686,7 @@ class Database:
         """Rename all memories from one namespace to another.
 
         Uses atomic batch update via merge_insert for data integrity.
+        On partial failure, attempts to rollback renamed records to original namespace.
 
         Args:
             old_namespace: Source namespace name.
@@ -1652,6 +1705,7 @@ class Database:
         old_namespace = _validate_namespace(old_namespace)
         new_namespace = _validate_namespace(new_namespace)
         safe_old = _sanitize_string(old_namespace)
+        safe_new = _sanitize_string(new_namespace)
 
         try:
             # Check if source namespace exists
@@ -1665,6 +1719,9 @@ class Database:
                 logger.debug(f"Namespace '{old_namespace}' renamed to itself ({count} records)")
                 return count
 
+            # Track renamed IDs for rollback capability
+            renamed_ids: list[str] = []
+
             # Fetch all records in batches with iteration safeguards
             batch_size = 1000
             max_iterations = 10000  # Safety cap: 10M records at 1000/batch
@@ -1693,6 +1750,9 @@ class Database:
                 if not records:
                     break
 
+                # Track IDs in this batch for potential rollback
+                batch_ids = [r["id"] for r in records]
+
                 # Update namespace field
                 for r in records:
                     r["namespace"] = new_namespace
@@ -1702,13 +1762,41 @@ class Database:
                     if isinstance(r.get("vector"), np.ndarray):
                         r["vector"] = r["vector"].tolist()
 
-                # Atomic upsert
-                (
-                    self.table.merge_insert("id")
-                    .when_matched_update_all()
-                    .when_not_matched_insert_all()
-                    .execute(records)
-                )
+                try:
+                    # Atomic upsert
+                    (
+                        self.table.merge_insert("id")
+                        .when_matched_update_all()
+                        .when_not_matched_insert_all()
+                        .execute(records)
+                    )
+                    # Only track as renamed after successful update
+                    renamed_ids.extend(batch_ids)
+                except Exception as batch_error:
+                    # Batch failed - attempt rollback of previously renamed records
+                    if renamed_ids:
+                        logger.warning(
+                            f"Batch {iteration} failed, attempting rollback of "
+                            f"{len(renamed_ids)} previously renamed records"
+                        )
+                        rollback_error = self._rollback_namespace_rename(
+                            renamed_ids, old_namespace
+                        )
+                        if rollback_error:
+                            raise StorageError(
+                                f"Namespace rename failed at batch {iteration} and "
+                                f"rollback also failed. {len(renamed_ids)} records may be "
+                                f"in inconsistent state (partially in '{new_namespace}'). "
+                                f"Original error: {batch_error}. Rollback error: {rollback_error}"
+                            ) from batch_error
+                        else:
+                            logger.info(
+                                f"Rollback successful, reverted {len(renamed_ids)} records "
+                                f"back to namespace '{old_namespace}'"
+                            )
+                    raise StorageError(
+                        f"Failed to rename namespace (rolled back): {batch_error}"
+                    ) from batch_error
 
                 updated += len(records)
 
@@ -1731,6 +1819,66 @@ class Database:
         except Exception as e:
             raise StorageError(f"Failed to rename namespace: {e}") from e
 
+    def _rollback_namespace_rename(
+        self, memory_ids: list[str], target_namespace: str
+    ) -> Exception | None:
+        """Attempt to revert renamed records back to original namespace.
+
+        Args:
+            memory_ids: List of memory IDs to revert.
+            target_namespace: Namespace to revert records to.
+
+        Returns:
+            None if rollback succeeded, Exception if it failed.
+        """
+        try:
+            if not memory_ids:
+                return None
+
+            safe_namespace = _sanitize_string(target_namespace)
+            now = utc_now()
+
+            # Process in batches for large rollbacks
+            batch_size = 1000
+            for i in range(0, len(memory_ids), batch_size):
+                batch_ids = memory_ids[i:i + batch_size]
+                id_list = ", ".join(f"'{_sanitize_string(mid)}'" for mid in batch_ids)
+
+                # Fetch records that need rollback
+                records = (
+                    self.table.search()
+                    .where(f"id IN ({id_list})")
+                    .to_list()
+                )
+
+                if not records:
+                    continue
+
+                # Revert namespace
+                for r in records:
+                    r["namespace"] = target_namespace
+                    r["updated_at"] = now
+                    if isinstance(r.get("metadata"), dict):
+                        r["metadata"] = json.dumps(r["metadata"])
+                    if isinstance(r.get("vector"), np.ndarray):
+                        r["vector"] = r["vector"].tolist()
+
+                # Atomic upsert to restore original namespace
+                (
+                    self.table.merge_insert("id")
+                    .when_matched_update_all()
+                    .when_not_matched_insert_all()
+                    .execute(records)
+                )
+
+            self._invalidate_namespace_cache()
+            logger.debug(f"Rolled back {len(memory_ids)} records to namespace '{target_namespace}'")
+            return None
+
+        except Exception as e:
+            logger.error(f"Namespace rename rollback failed: {e}")
+            return e
+
     @with_stale_connection_recovery
     def get_stats(self, namespace: str | None = None) -> dict[str, Any]:
         """Get comprehensive database statistics.
@@ -1827,15 +1975,18 @@ class Database:
         safe_ns = _sanitize_string(namespace)
 
         try:
-            # Get records for this namespace (select created_at and content for stats)
-            records = (
+            # Get count efficiently
+            filter_expr = f"namespace = '{safe_ns}'"
+            count_results = (
                 self.table.search()
-                .where(f"namespace = '{safe_ns}'")
-                .select(["created_at", "content"])
+                .where(filter_expr)
+                .select(["id"])
+                .limit(1000000)  # High limit to count all
                 .to_list()
             )
+            memory_count = len(count_results)
 
-            if not records:
+            if memory_count == 0:
                 return {
                     "namespace": namespace,
                     "memory_count": 0,
@@ -1844,18 +1995,42 @@ class Database:
                     "avg_content_length": None,
                 }
 
-            # Find oldest and newest
-            created_times = [r["created_at"] for r in records]
-            oldest = min(created_times)
-            newest = max(created_times)
+            # Get oldest memory (sort ascending, limit 1)
+            oldest_records = (
+                self.table.search()
+                .where(filter_expr)
+                .select(["created_at"])
+                .limit(1)
+                .to_list()
+            )
+            oldest = oldest_records[0]["created_at"] if oldest_records else None
 
-            # Calculate average content length
-            content_lengths = [len(r.get("content", "")) for r in records]
-            avg_content_length = sum(content_lengths) / len(content_lengths)
+            # Get newest memory - need to fetch more and find max since LanceDB
+            # doesn't support ORDER BY DESC efficiently
+            # Sample up to 1000 records for stats to avoid loading everything
+            sample_size = min(memory_count, 1000)
+            sample_records = (
+                self.table.search()
+                .where(filter_expr)
+                .select(["created_at", "content"])
+                .limit(sample_size)
+                .to_list()
+            )
+
+            # Find newest from sample (for large namespaces this is approximate)
+            if sample_records:
+                created_times = [r["created_at"] for r in sample_records]
+                newest = max(created_times)
+                # Calculate average content length from sample
+                content_lengths = [len(r.get("content", "")) for r in sample_records]
+                avg_content_length = sum(content_lengths) / len(content_lengths)
+            else:
+                newest = oldest
+                avg_content_length = None
 
             return {
                 "namespace": namespace,
-                "memory_count": len(records),
+                "memory_count": memory_count,
                 "oldest_memory": oldest,
                 "newest_memory": newest,
                 "avg_content_length": avg_content_length,
@@ -2015,21 +2190,23 @@ class Database:
 
     @with_process_lock
     @with_write_lock
-    def delete_batch(self, memory_ids: list[str]) -> int:
+    def delete_batch(self, memory_ids: list[str]) -> tuple[int, list[str]]:
         """Delete multiple memories atomically using IN clause.
 
         Args:
             memory_ids: List of memory UUIDs to delete.
 
         Returns:
-            Number of memories actually deleted.
+            Tuple of (count_deleted, list_of_deleted_ids) where:
+                - count_deleted: Number of memories actually deleted
+                - list_of_deleted_ids: IDs that were actually deleted
 
         Raises:
             ValidationError: If any memory_id is invalid.
             StorageError: If database operation fails.
         """
         if not memory_ids:
-            return 0
+            return (0, [])
 
         # Validate all IDs first (fail fast)
         validated_ids: list[str] = []
@@ -2038,21 +2215,32 @@ class Database:
             validated_ids.append(_sanitize_string(validated_id))
 
         try:
-            count_before: int = self.table.count_rows()
-
-            # Build IN clause for atomic batch delete
+            # First, check which IDs actually exist
             id_list = ", ".join(f"'{mid}'" for mid in validated_ids)
             filter_expr = f"id IN ({id_list})"
-            self.table.delete(filter_expr)
+            existing_records = (
+                self.table.search()
+                .where(filter_expr)
+                .select(["id"])
+                .limit(len(validated_ids))
+                .to_list()
+            )
+            existing_ids = [r["id"] for r in existing_records]
+
+            if not existing_ids:
+                return (0, [])
+
+            # Delete only existing IDs
+            existing_id_list = ", ".join(f"'{mid}'" for mid in existing_ids)
+            delete_expr = f"id IN ({existing_id_list})"
+            self.table.delete(delete_expr)
 
             self._invalidate_count_cache()
+            self._track_modification()
             self._invalidate_namespace_cache()
 
-            count_after: int = self.table.count_rows()
-            deleted = count_before - count_after
-
-            logger.debug(f"Batch deleted {deleted} memories")
-            return deleted
+            logger.debug(f"Batch deleted {len(existing_ids)} memories")
+            return (len(existing_ids), existing_ids)
         except ValidationError:
             raise
         except Exception as e:
@@ -2150,6 +2338,10 @@ class Database:
             backoff=self.retry_backoff_seconds,
         )
 
+    # ========================================================================
+    # Search Operations (delegates to SearchManager)
+    # ========================================================================
+
     def _calculate_search_params(
         self,
         count: int,
@@ -2157,59 +2349,12 @@ class Database:
         nprobes_override: int | None = None,
         refine_factor_override: int | None = None,
     ) -> tuple[int, int]:
-        """Calculate optimal search parameters based on dataset size and limit.
-
-        Dynamically tunes nprobes and refine_factor for optimal recall/speed tradeoff.
-
-        Args:
-            count: Number of rows in the dataset.
-            limit: Number of results requested.
-            nprobes_override: Optional override for nprobes (uses this if provided).
-            refine_factor_override: Optional override for refine_factor.
-
-        Returns:
-            Tuple of (nprobes, refine_factor).
-
-        Scaling rules:
-            - nprobes: Base from config, scaled up for larger datasets
-              - <100K: config value (default 20)
-              - 100K-1M: max(config, 30)
-              - 1M-10M: max(config, 50)
-              - >10M: max(config, 100)
-            - refine_factor: Base from config, scaled up for small limits
-              - limit <= 5: config value * 2
-              - limit <= 20: config value
-              - limit > 20: max(config // 2, 2)
-        """
-        # Calculate nprobes based on dataset size
-        if nprobes_override is not None:
-            nprobes = nprobes_override
-        else:
-            base_nprobes = self.index_nprobes
-            if count < 100_000:
-                nprobes = base_nprobes
-            elif count < 1_000_000:
-                nprobes = max(base_nprobes, 30)
-            elif count < 10_000_000:
-                nprobes = max(base_nprobes, 50)
-            else:
-                nprobes = max(base_nprobes, 100)
-
-        # Calculate refine_factor based on limit
-        if refine_factor_override is not None:
-            refine_factor = refine_factor_override
-        else:
-            base_refine = self.index_refine_factor
-            if limit <= 5:
-                # Small limits need more refinement for accuracy
-                refine_factor = base_refine * 2
-            elif limit <= 20:
-                refine_factor = base_refine
-            else:
-                # Large limits can use less refinement
-                refine_factor = max(base_refine // 2, 2)
-
-        return nprobes, refine_factor
+        """Calculate optimal search parameters. Delegates to SearchManager."""
+        if self._search_manager is None:
+            raise StorageError("Database not connected")
+        return self._search_manager.calculate_search_params(
+            count, limit, nprobes_override, refine_factor_override
+        )
 
     @with_stale_connection_recovery
     @retry_on_storage_error(max_attempts=3, backoff=0.5)
@@ -2223,19 +2368,16 @@ class Database:
         refine_factor: int | None = None,
         include_vector: bool = False,
     ) -> list[dict[str, Any]]:
-        """Search for similar memories by vector with performance tuning.
+        """Search for similar memories by vector. Delegates to SearchManager.
 
         Args:
             query_vector: Query embedding vector.
             limit: Maximum number of results.
             namespace: Filter to specific namespace.
             min_similarity: Minimum similarity threshold (0-1).
-            nprobes: Number of partitions to search (higher = better recall).
-                     Only effective when vector index exists. Defaults to dynamic calculation.
+            nprobes: Number of partitions to search.
             refine_factor: Re-rank top (refine_factor * limit) for accuracy.
-                          Defaults to dynamic calculation based on limit.
             include_vector: Whether to include vector embeddings in results.
-                           Defaults to False to reduce response size.
 
         Returns:
             List of memory records with similarity scores.
@@ -2244,66 +2386,53 @@ class Database:
             ValidationError: If input validation fails.
             StorageError: If database operation fails.
         """
-        try:
-            search = self.table.search(query_vector.tolist())
+        if self._search_manager is None:
+            raise StorageError("Database not connected")
+        return self._search_manager.vector_search(
+            query_vector=query_vector,
+            limit=limit,
+            namespace=namespace,
+            min_similarity=min_similarity,
+            nprobes=nprobes,
+            refine_factor=refine_factor,
+            include_vector=include_vector,
+        )
 
-            # Distance type for queries (cosine for semantic similarity)
-            # Note: When vector index exists, the index's metric is used
-            search = search.distance_type("cosine")
+    @with_stale_connection_recovery
+    @retry_on_storage_error(max_attempts=3, backoff=0.5)
+    def batch_vector_search_native(
+        self,
+        query_vectors: list[np.ndarray],
+        limit_per_query: int = 3,
+        namespace: str | None = None,
+        min_similarity: float = 0.0,
+        include_vector: bool = False,
+    ) -> list[list[dict[str, Any]]]:
+        """Batch search using native LanceDB. Delegates to SearchManager.
 
-            # Apply performance tuning when index exists (use cached count)
-            count = self._get_cached_row_count()
-            if count > self.vector_index_threshold and self._has_vector_index:
-                # Use dynamic calculation for search params
-                actual_nprobes, actual_refine = self._calculate_search_params(
-                    count, limit, nprobes, refine_factor
-                )
-                search = search.nprobes(actual_nprobes)
-                search = search.refine_factor(actual_refine)
+        Args:
+            query_vectors: List of query embedding vectors.
+            limit_per_query: Maximum number of results per query.
+            namespace: Filter to specific namespace.
+            min_similarity: Minimum similarity threshold (0-1).
+            include_vector: Whether to include vector embeddings in results.
 
-            # Build filter with sanitized namespace
-            # prefilter=True applies namespace filter BEFORE vector search for better performance
-            if namespace:
-                namespace = _validate_namespace(namespace)
-                safe_ns = _sanitize_string(namespace)
-                search = search.where(f"namespace = '{safe_ns}'", prefilter=True)
-
-            # Vector projection: exclude vector column to reduce response size
-            if not include_vector:
-                search = search.select([
-                    "id", "content", "namespace", "metadata",
-                    "created_at", "updated_at", "last_accessed",
-                    "importance", "tags", "source", "access_count",
-                    "expires_at",
-                ])
-
-            # Fetch extra if filtering by similarity
-            fetch_limit = limit * 2 if min_similarity > 0.0 else limit
-            results: list[dict[str, Any]] = search.limit(fetch_limit).to_list()
-
-            # Process results
-            filtered_results: list[dict[str, Any]] = []
-            for record in results:
-                record["metadata"] = json.loads(record["metadata"]) if record["metadata"] else {}
-                # LanceDB returns _distance, convert to similarity
-                if "_distance" in record:
-                    # Cosine distance to similarity: 1 - distance
-                    # Clamp to [0, 1] (cosine distance can exceed 1 for unnormalized)
-                    similarity = max(0.0, min(1.0, 1 - record["_distance"]))
-                    record["similarity"] = similarity
-                    del record["_distance"]
-
-                # Apply similarity threshold
-                if record.get("similarity", 0) >= min_similarity:
-                    filtered_results.append(record)
-                    if len(filtered_results) >= limit:
-                        break
-
-            return filtered_results
-        except ValidationError:
-            raise
-        except Exception as e:
-            raise StorageError(f"Failed to search: {e}") from e
+        Returns:
+            List of result lists, one per query vector.
+
+        Raises:
+            ValidationError: If input validation fails.
+            StorageError: If database operation fails.
+        """
+        if self._search_manager is None:
+            raise StorageError("Database not connected")
+        return self._search_manager.batch_vector_search_native(
+            query_vectors=query_vectors,
+            limit_per_query=limit_per_query,
+            namespace=namespace,
+            min_similarity=min_similarity,
+            include_vector=include_vector,
+        )
 
     @with_stale_connection_recovery
     @retry_on_storage_error(max_attempts=3, backoff=0.5)
@@ -2316,10 +2445,7 @@ class Database:
         alpha: float = 0.5,
         min_similarity: float = 0.0,
     ) -> list[dict[str, Any]]:
-        """Hybrid search combining vector similarity and keyword matching.
-
-        Uses LinearCombinationReranker to balance vector and keyword scores
-        based on the alpha parameter.
+        """Hybrid search combining vector and keyword. Delegates to SearchManager.
 
         Args:
             query: Text query for full-text search.
@@ -2327,9 +2453,7 @@ class Database:
             limit: Number of results.
             namespace: Filter to namespace.
             alpha: Balance between vector (1.0) and keyword (0.0).
-                   0.5 = balanced (recommended).
-            min_similarity: Minimum similarity threshold (0.0-1.0).
-                           Results below this threshold are filtered out.
+            min_similarity: Minimum similarity threshold.
 
         Returns:
             List of memory records with combined scores.
@@ -2338,80 +2462,16 @@ class Database:
             ValidationError: If input validation fails.
             StorageError: If database operation fails.
         """
-        try:
-            # Check if FTS is available
-            if not self._has_fts_index:
-                logger.debug("FTS index not available, falling back to vector search")
-                return self.vector_search(query_vector, limit=limit, namespace=namespace)
-
-            # Create hybrid search with explicit vector column specification
-            # Required when using external embeddings (not LanceDB built-in)
-            search = (
-                self.table.search(query, query_type="hybrid")
-                .vector(query_vector.tolist())
-                .vector_column_name("vector")
-            )
-
-            # Apply alpha parameter using LinearCombinationReranker
-            # alpha=1.0 means full vector, alpha=0.0 means full FTS
-            try:
-                from lancedb.rerankers import LinearCombinationReranker
-
-                reranker = LinearCombinationReranker(weight=alpha)
-                search = search.rerank(reranker)
-            except ImportError:
-                logger.debug("LinearCombinationReranker not available, using default reranking")
-            except Exception as e:
-                logger.debug(f"Could not apply reranker: {e}")
-
-            # Apply namespace filter
-            if namespace:
-                namespace = _validate_namespace(namespace)
-                safe_ns = _sanitize_string(namespace)
-                search = search.where(f"namespace = '{safe_ns}'")
-
-            results: list[dict[str, Any]] = search.limit(limit).to_list()
-
-            # Process results - normalize scores and clean up internal columns
-            processed_results: list[dict[str, Any]] = []
-            for record in results:
-                record["metadata"] = json.loads(record["metadata"]) if record["metadata"] else {}
-
-                # Compute similarity from various score columns
-                # Priority: _relevance_score > _distance > _score > default
-                similarity: float
-                if "_relevance_score" in record:
-                    # Reranker output - use directly (already 0-1 range)
-                    similarity = float(record["_relevance_score"])
-                    del record["_relevance_score"]
-                elif "_distance" in record:
-                    # Vector distance - convert to similarity
-                    similarity = max(0.0, min(1.0, 1 - float(record["_distance"])))
-                    del record["_distance"]
-                elif "_score" in record:
-                    # BM25 score - normalize using score/(1+score)
-                    score = float(record["_score"])
-                    similarity = score / (1.0 + score)
-                    del record["_score"]
-                else:
-                    # No score column - use default
-                    similarity = 0.5
-
-                record["similarity"] = similarity
-
-                # Mark as hybrid result with alpha value
-                record["search_type"] = "hybrid"
-                record["alpha"] = alpha
-
-                # Apply min_similarity filter
-                if similarity >= min_similarity:
-                    processed_results.append(record)
-
-            return processed_results
-
-        except Exception as e:
-            logger.warning(f"Hybrid search failed, falling back to vector search: {e}")
-            return self.vector_search(query_vector, limit=limit, namespace=namespace)
+        if self._search_manager is None:
+            raise StorageError("Database not connected")
+        return self._search_manager.hybrid_search(
+            query=query,
+            query_vector=query_vector,
+            limit=limit,
+            namespace=namespace,
+            alpha=alpha,
+            min_similarity=min_similarity,
+        )
 
     @with_stale_connection_recovery
     @retry_on_storage_error(max_attempts=3, backoff=0.5)
@@ -2420,20 +2480,19 @@ class Database:
         query_vectors: list[np.ndarray],
         limit_per_query: int = 3,
         namespace: str | None = None,
-        parallel: bool = False,
-        max_workers: int = 4,
+        parallel: bool = False,  # Deprecated
+        max_workers: int = 4,  # Deprecated
+        include_vector: bool = False,
     ) -> list[list[dict[str, Any]]]:
-        """Search for similar memories using multiple query vectors.
-
-        Efficient for operations like journey interpolation where multiple
-        points need to find nearby memories.
+        """Search using multiple query vectors. Delegates to SearchManager.
 
         Args:
             query_vectors: List of query embedding vectors.
             limit_per_query: Maximum results per query vector.
             namespace: Filter to specific namespace.
-            parallel: Execute searches in parallel using ThreadPoolExecutor.
-            max_workers: Maximum worker threads for parallel execution.
+            parallel: Deprecated, kept for backward compatibility.
+            max_workers: Deprecated, kept for backward compatibility.
+            include_vector: Whether to include vector embeddings in results.
 
         Returns:
             List of result lists (one per query vector).
@@ -2441,52 +2500,16 @@ class Database:
         Raises:
             StorageError: If database operation fails.
         """
-        if not query_vectors:
-            return []
-
-        # Build namespace filter once
-        where_clause: str | None = None
-        if namespace:
-            namespace = _validate_namespace(namespace)
-            safe_ns = _sanitize_string(namespace)
-            where_clause = f"namespace = '{safe_ns}'"
-
-        def search_single(vec: np.ndarray) -> list[dict[str, Any]]:
-            """Execute a single vector search."""
-            search = self.table.search(vec.tolist()).distance_type("cosine")
-
-            if where_clause:
-                search = search.where(where_clause)
-
-            results: list[dict[str, Any]] = search.limit(limit_per_query).to_list()
-
-            # Process results
-            for record in results:
-                meta = record["metadata"]
-                record["metadata"] = json.loads(meta) if meta else {}
-                if "_distance" in record:
-                    record["similarity"] = max(0.0, min(1.0, 1 - record["_distance"]))
-                    del record["_distance"]
-
-            return results
-
-        try:
-            if parallel and len(query_vectors) > 1:
-                # Use ThreadPoolExecutor for parallel execution
-                from concurrent.futures import ThreadPoolExecutor
-
-                workers = min(max_workers, len(query_vectors))
-                with ThreadPoolExecutor(max_workers=workers) as executor:
-                    # Map preserves order
-                    all_results = list(executor.map(search_single, query_vectors))
-            else:
-                # Sequential execution
-                all_results = [search_single(vec) for vec in query_vectors]
-
-            return all_results
-
-        except Exception as e:
-            raise StorageError(f"Batch vector search failed: {e}") from e
+        if self._search_manager is None:
+            raise StorageError("Database not connected")
+        return self._search_manager.batch_vector_search(
+            query_vectors=query_vectors,
+            limit_per_query=limit_per_query,
+            namespace=namespace,
+            parallel=parallel,
+            max_workers=max_workers,
+            include_vector=include_vector,
+        )
 
     def get_vectors_for_clustering(
         self,
@@ -2932,6 +2955,7 @@ class Database:
 
             if deleted > 0:
                 self._invalidate_count_cache()
+                self._track_modification(deleted)
                 self._invalidate_namespace_cache()
                 logger.info(f"Cleaned up {deleted} expired memories")
 
@@ -2940,108 +2964,106 @@ class Database:
             raise StorageError(f"Failed to cleanup expired memories: {e}") from e
 
     # ========================================================================
-    # Snapshot / Version Management
+    # Snapshot / Version Management (delegated to VersionManager)
     # ========================================================================
 
     def create_snapshot(self, tag: str) -> int:
         """Create a named snapshot of the current table state.
 
-        LanceDB automatically versions data on every write. This method
-        returns the current version number which can be used with restore_snapshot().
-
-        Args:
-            tag: Semantic version tag (e.g., "v1.0.0", "backup-2024-01").
-                 Note: Tag is logged for reference but LanceDB tracks versions
-                 numerically. Consider storing tag->version mappings externally
-                 if tag-based retrieval is needed.
-
-        Returns:
-            Version number of the snapshot.
-
-        Raises:
-            StorageError: If snapshot creation fails.
+        Delegates to VersionManager. See VersionManager.create_snapshot for details.
         """
-        try:
-            version = self.table.version
-            logger.info(f"Created snapshot '{tag}' at version {version}")
-            return version
-        except Exception as e:
-            raise StorageError(f"Failed to create snapshot: {e}") from e
+        if self._version_manager is None:
+            raise StorageError("Database not connected")
+        return self._version_manager.create_snapshot(tag)
 
     def list_snapshots(self) -> list[dict[str, Any]]:
         """List available versions/snapshots.
 
-        Returns:
-            List of version information dictionaries. Each dict contains
-            at minimum 'version' key. Additional fields depend on LanceDB
-            version and available metadata.
+        Delegates to VersionManager. See VersionManager.list_snapshots for details.
+        """
+        if self._version_manager is None:
+            raise StorageError("Database not connected")
+        return self._version_manager.list_snapshots()
 
-        Raises:
-            StorageError: If listing fails.
+    def restore_snapshot(self, version: int) -> None:
+        """Restore table to a specific version.
+
+        Delegates to VersionManager. See VersionManager.restore_snapshot for details.
         """
-        try:
-            versions_info: list[dict[str, Any]] = []
+        if self._version_manager is None:
+            raise StorageError("Database not connected")
+        self._version_manager.restore_snapshot(version)
 
-            # Try to get version history if available
-            if hasattr(self.table, "list_versions"):
-                try:
-                    versions = self.table.list_versions()
-                    for v in versions:
-                        if isinstance(v, dict):
-                            versions_info.append(v)
-                        elif hasattr(v, "version"):
-                            versions_info.append({
-                                "version": v.version,
-                                "timestamp": getattr(v, "timestamp", None),
-                            })
-                        else:
-                            versions_info.append({"version": v})
-                except Exception as e:
-                    logger.debug(f"list_versions not fully supported: {e}")
+    def get_current_version(self) -> int:
+        """Get the current table version number.
 
-            # Always include current version
-            if not versions_info:
-                versions_info.append({"version": self.table.version})
+        Delegates to VersionManager. See VersionManager.get_current_version for details.
+        """
+        if self._version_manager is None:
+            raise StorageError("Database not connected")
+        return self._version_manager.get_current_version()
 
-            return versions_info
-        except Exception as e:
-            logger.warning(f"Could not list snapshots: {e}")
-            return [{"version": 0, "error": str(e)}]
+    # ========================================================================
+    # Idempotency Key Management (delegates to IdempotencyManager)
+    # ========================================================================
 
-    def restore_snapshot(self, version: int) -> None:
-        """Restore table to a specific version.
+    @property
+    def idempotency_table(self) -> LanceTable:
+        """Get the idempotency keys table. Delegates to IdempotencyManager."""
+        if self._idempotency_manager is None:
+            raise StorageError("Database not connected")
+        return self._idempotency_manager.idempotency_table
 
-        This creates a NEW version that reflects the old state
-        (doesn't delete history).
+    def get_by_idempotency_key(self, key: str) -> IdempotencyRecord | None:
+        """Look up an idempotency record by key. Delegates to IdempotencyManager.
 
         Args:
-            version: The version number to restore to.
+            key: The idempotency key to look up.
+
+        Returns:
+            IdempotencyRecord if found and not expired, None otherwise.
 
         Raises:
-            ValidationError: If version is invalid.
-            StorageError: If restore fails.
+            StorageError: If database operation fails.
         """
-        if version < 0:
-            raise ValidationError("Version must be non-negative")
+        if self._idempotency_manager is None:
+            raise StorageError("Database not connected")
+        return self._idempotency_manager.get_by_idempotency_key(key)
 
-        try:
-            self.table.restore(version)
-            self._invalidate_count_cache()
-            self._invalidate_namespace_cache()
-            logger.info(f"Restored to version {version}")
-        except Exception as e:
-            raise StorageError(f"Failed to restore snapshot: {e}") from e
+    @with_process_lock
+    @with_write_lock
+    def store_idempotency_key(
+        self,
+        key: str,
+        memory_id: str,
+        ttl_hours: float = 24.0,
+    ) -> None:
+        """Store an idempotency key mapping. Delegates to IdempotencyManager.
 
-    def get_current_version(self) -> int:
-        """Get the current table version number.
+        Args:
+            key: The idempotency key.
+            memory_id: The memory ID that was created.
+            ttl_hours: Time-to-live in hours (default: 24 hours).
+
+        Raises:
+            ValidationError: If inputs are invalid.
+            StorageError: If database operation fails.
+        """
+        if self._idempotency_manager is None:
+            raise StorageError("Database not connected")
+        self._idempotency_manager.store_idempotency_key(key, memory_id, ttl_hours)
+
+    @with_process_lock
+    @with_write_lock
+    def cleanup_expired_idempotency_keys(self) -> int:
+        """Remove expired idempotency keys. Delegates to IdempotencyManager.
 
         Returns:
-            Current version number.
+            Number of keys removed.
 
         Raises:
-            StorageError: If version cannot be retrieved.
+            StorageError: If cleanup fails.
         """
-        try:
-            return self.table.version
-        except Exception as e:
-            raise StorageError(f"Failed to get current version: {e}") from e
+        if self._idempotency_manager is None:
+            raise StorageError("Database not connected")
+        return self._idempotency_manager.cleanup_expired_idempotency_keys()