mcp-code-indexer 4.2.15__py3-none-any.whl → 4.2.17__py3-none-any.whl

mcp_code_indexer/database/database.py

@@ -27,13 +27,18 @@ from mcp_code_indexer.database.exceptions import (
 )
 from mcp_code_indexer.database.models import (
     FileDescription,
+    IndexMeta,
     Project,
     ProjectOverview,
     SearchResult,
+    SyncStatus,
     WordFrequencyResult,
     WordFrequencyTerm,
 )
-from mcp_code_indexer.database.retry_executor import create_retry_executor
+from mcp_code_indexer.database.retry_executor import (
+    create_retry_executor,
+    DatabaseLockError,
+)
 from mcp_code_indexer.query_preprocessor import preprocess_search_query
 
 logger = logging.getLogger(__name__)
@@ -52,7 +57,7 @@ class DatabaseManager:
         db_path: Path,
         pool_size: int = 3,
         retry_count: int = 5,
-        timeout: float = 10.0,
+        timeout: float = 30.0,
         enable_wal_mode: bool = True,
         health_check_interval: float = 30.0,
         retry_min_wait: float = 0.1,
@@ -220,7 +225,7 @@ class DatabaseManager:
                 "PRAGMA cache_size = -64000",  # 64MB cache
                 "PRAGMA temp_store = MEMORY",  # Use memory for temp tables
                 "PRAGMA mmap_size = 268435456",  # 256MB memory mapping
-                "PRAGMA busy_timeout = 10000",  # 10s timeout (reduced from 30s)
+                f"PRAGMA busy_timeout = {int(self.timeout * 1000)}",  # Use configured timeout
                 "PRAGMA optimize",  # Enable query planner optimizations
             ]
         )
@@ -315,12 +320,10 @@ class DatabaseManager:
         self, operation_name: str = "write_operation"
     ) -> AsyncIterator[aiosqlite.Connection]:
         """
-        Get a database connection with write serialization and automatic
-        retry logic.
+        Get a database connection with write serialization.
 
-        This uses the new RetryExecutor to properly handle retry logic
-        without the broken yield-in-retry-loop pattern that caused
-        generator errors.
+        Ensures the write lock is held throughout the duration of the context
+        to prevent race conditions and database locking errors.
 
         Args:
             operation_name: Name of the operation for logging and
@@ -331,43 +334,10 @@ class DatabaseManager:
                 "DatabaseManager not initialized - call initialize() first"
             )
 
-        async def get_write_connection() -> aiosqlite.Connection:
-            """Inner function to get connection - retried by executor."""
-            if self._write_lock is None:
-                raise RuntimeError("Write lock not initialized")
-            async with self._write_lock:
-                async with self.get_connection() as conn:
-                    return conn
-
-        try:
-            # Use retry executor to handle connection acquisition with retries
-            connection = await self._retry_executor.execute_with_retry(
-                get_write_connection, operation_name
-            )
-
-            try:
-                yield connection
-
-                # Success - retry executor handles all failure tracking
-
-            except Exception:
-                # Error handling is managed by the retry executor
-                raise
-
-        except DatabaseError:
-            # Re-raise our custom database errors as-is
-            raise
-        except Exception as e:
-            # Classify and wrap other exceptions
-            classified_error = classify_sqlite_error(e, operation_name)
-            logger.error(
-                (
-                    f"Database operation '{operation_name}' failed: "
-                    f"{classified_error.message}"
-                ),
-                extra={"structured_data": classified_error.to_dict()},
-            )
-            raise classified_error
+        # Acquire lock for exclusive write access - hold it for entire context
+        async with self._write_lock:
+            async with self.get_connection() as conn:
+                yield conn
 
     def get_database_stats(self) -> Dict[str, Any]:
         """
@@ -523,26 +493,39 @@ class DatabaseManager:
 
                 return result
 
-            except (aiosqlite.OperationalError, asyncio.TimeoutError) as e:
+            except aiosqlite.OperationalError as e:
                 # Record locking event for metrics
-                if self._metrics_collector and "locked" in str(e).lower():
+                error_msg = str(e).lower()
+                if self._metrics_collector and "locked" in error_msg:
                     self._metrics_collector.record_locking_event(operation_name, str(e))
 
-                # Classify the error for better handling
-                classified_error = classify_sqlite_error(e, operation_name)
-
-                # Record failed operation metrics for non-retryable errors
-                if not is_retryable_error(classified_error):
-                    if self._metrics_collector:
-                        self._metrics_collector.record_operation(
-                            operation_name,
-                            timeout_seconds * 1000,
-                            False,
-                            len(self._connection_pool),
-                        )
+                # For retryable errors (locked/busy), re-raise the ORIGINAL error
+                # so tenacity can retry. Only classify non-retryable errors.
+                if "locked" in error_msg or "busy" in error_msg:
+                    raise  # Let tenacity retry this
 
+                # Non-retryable OperationalError - classify and raise
+                classified_error = classify_sqlite_error(e, operation_name)
+                if self._metrics_collector:
+                    self._metrics_collector.record_operation(
+                        operation_name,
+                        timeout_seconds * 1000,
+                        False,
+                        len(self._connection_pool),
+                    )
                 raise classified_error
 
+            except asyncio.TimeoutError as e:
+                # Timeout on BEGIN IMMEDIATE - this is retryable
+                if self._metrics_collector:
+                    self._metrics_collector.record_locking_event(
+                        operation_name, "timeout waiting for lock"
+                    )
+                # Re-raise as OperationalError so tenacity can retry
+                raise aiosqlite.OperationalError(
+                    f"Timeout waiting for database lock: {e}"
+                ) from e
+
         try:
             # Create a temporary retry executor with custom max_retries if different
             # from default
@@ -567,8 +550,27 @@ class DatabaseManager:
                     execute_transaction, operation_name
                 )
 
+        except DatabaseLockError as e:
+            # Retries exhausted - record metrics and convert to DatabaseError
+            if self._metrics_collector:
+                self._metrics_collector.record_operation(
+                    operation_name,
+                    timeout_seconds * 1000,
+                    False,
+                    len(self._connection_pool),
+                )
+            # Convert to a proper DatabaseError for consistent error handling
+            raise DatabaseError(
+                f"Database operation failed after retries: {e.message}",
+                error_context={
+                    "operation": operation_name,
+                    "retry_count": e.retry_count,
+                    "retryable": False,  # Retries already exhausted
+                },
+            ) from e
+
         except DatabaseError:
-            # Record failed operation metrics for final failure
+            # Non-retryable DatabaseError from classification
             if self._metrics_collector:
                 self._metrics_collector.record_operation(
                     operation_name,
@@ -740,6 +742,25 @@ class DatabaseManager:
             await db.commit()
             logger.debug(f"Updated project: {project.id}")
 
+    async def set_project_vector_mode(self, project_id: str, enabled: bool) -> None:
+        """Set the vector_mode for a specific project."""
+        async with self.get_write_connection_with_retry(
+            "set_project_vector_mode"
+        ) as db:
+            await db.execute(
+                "UPDATE projects SET vector_mode = ? WHERE id = ?",
+                (int(enabled), project_id),
+            )
+
+            # Check if the project was actually updated
+            cursor = await db.execute("SELECT changes()")
+            changes = await cursor.fetchone()
+            if changes[0] == 0:
+                raise DatabaseError(f"Project not found: {project_id}")
+
+            await db.commit()
+            logger.debug(f"Set vector_mode={enabled} for project: {project_id}")
+
     async def get_all_projects(self) -> List[Project]:
         """Get all projects in the database."""
         async with self.get_connection() as db:
@@ -751,12 +772,18 @@ class DatabaseManager:
             projects = []
             for row in rows:
                 aliases = json.loads(row[2]) if row[2] else []
+                created = row[3]
+                last_accessed = row[4]
+                if isinstance(created, str):
+                    created = datetime.fromisoformat(created)
+                if isinstance(last_accessed, str):
+                    last_accessed = datetime.fromisoformat(last_accessed)
                 project = Project(
                     id=row[0],
                     name=row[1],
                     aliases=aliases,
-                    created=row[3],
-                    last_accessed=row[4],
+                    created=created,
+                    last_accessed=last_accessed,
                     vector_mode=bool(row[5]),
                 )
                 projects.append(project)
@@ -774,12 +801,18 @@ class DatabaseManager:
             projects = []
             for row in rows:
                 aliases = json.loads(row[2]) if row[2] else []
+                created = row[3]
+                last_accessed = row[4]
+                if isinstance(created, str):
+                    created = datetime.fromisoformat(created)
+                if isinstance(last_accessed, str):
+                    last_accessed = datetime.fromisoformat(last_accessed)
                 project = Project(
                     id=row[0],
                     name=row[1],
                     aliases=aliases,
-                    created=row[3],
-                    last_accessed=row[4],
+                    created=created,
+                    last_accessed=last_accessed,
                     vector_mode=bool(row[5]),
                 )
                 projects.append(project)
@@ -790,17 +823,22 @@ class DatabaseManager:
 
     async def create_file_description(self, file_desc: FileDescription) -> None:
         """Create or update a file description."""
-        async with self.get_write_connection_with_retry(
-            "create_file_description"
-        ) as db:
+        async def operation(db: aiosqlite.Connection) -> None:
             await db.execute(
                 """
-                INSERT OR REPLACE INTO file_descriptions
+                INSERT INTO file_descriptions
                 (
                     project_id, file_path, description, file_hash, last_modified,
                     version, source_project_id, to_be_cleaned
                 )
                 VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+                ON CONFLICT(project_id, file_path) DO UPDATE SET
+                    description=excluded.description,
+                    file_hash=excluded.file_hash,
+                    last_modified=excluded.last_modified,
+                    version=excluded.version,
+                    source_project_id=excluded.source_project_id,
+                    to_be_cleaned=excluded.to_be_cleaned
                 """,
                 (
                     file_desc.project_id,
@@ -813,8 +851,12 @@ class DatabaseManager:
                     file_desc.to_be_cleaned,
                 ),
             )
-            await db.commit()
-            logger.debug(f"Saved file description: {file_desc.file_path}")
+
+        await self.execute_transaction_with_retry(
+            operation,
+            "create_file_description"
+        )
+        logger.debug(f"Saved file description: {file_desc.file_path}")
 
     async def get_file_description(
         self, project_id: str, file_path: str
@@ -898,12 +940,19 @@ class DatabaseManager:
 
             await conn.executemany(
                 """
-                INSERT OR REPLACE INTO file_descriptions
+                INSERT INTO file_descriptions
                 (
                     project_id, file_path, description, file_hash, last_modified,
                     version, source_project_id, to_be_cleaned
                 )
                 VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+                ON CONFLICT(project_id, file_path) DO UPDATE SET
+                    description=excluded.description,
+                    file_hash=excluded.file_hash,
+                    last_modified=excluded.last_modified,
+                    version=excluded.version,
+                    source_project_id=excluded.source_project_id,
+                    to_be_cleaned=excluded.to_be_cleaned
                 """,
                 data,
             )
@@ -1018,7 +1067,7 @@ class DatabaseManager:
 
     async def create_project_overview(self, overview: ProjectOverview) -> None:
         """Create or update a project overview."""
-        async with self.get_write_connection() as db:
+        async def operation(db: aiosqlite.Connection) -> None:
             await db.execute(
                 """
                 INSERT OR REPLACE INTO project_overviews
@@ -1033,8 +1082,12 @@ class DatabaseManager:
                     overview.total_tokens,
                 ),
             )
-            await db.commit()
-            logger.debug(f"Created/updated overview for project {overview.project_id}")
+
+        await self.execute_transaction_with_retry(
+            operation,
+            "create_project_overview"
+        )
+        logger.debug(f"Created/updated overview for project {overview.project_id}")
 
     async def get_project_overview(self, project_id: str) -> Optional[ProjectOverview]:
         """Get project overview by ID."""
@@ -1067,10 +1120,8 @@ class DatabaseManager:
         Returns:
             List of file paths that were marked for cleanup
         """
-        removed_files: List[str] = []
-
-        async def cleanup_operation(conn: aiosqlite.Connection) -> List[str]:
-            # Get all active file descriptions for this project
+        # 1. Get all active file paths (fast DB read)
+        async with self.get_connection() as conn:
             cursor = await conn.execute(
                 (
                     "SELECT file_path FROM file_descriptions WHERE "
@@ -1078,46 +1129,29 @@ class DatabaseManager:
                 ),
                 (project_id,),
             )
-
             rows = await cursor.fetchall()
+            file_paths = [row["file_path"] for row in rows]
 
-            # Check which files no longer exist
-            to_remove = []
-            for row in rows:
-                file_path = row["file_path"]
+        # 2. Check existence on disk (blocking IO - run in executor)
+        def find_removed_files() -> List[str]:
+            missing = []
+            for file_path in file_paths:
                 full_path = project_root / file_path
-
                 if not full_path.exists():
-                    to_remove.append(file_path)
-
-            # Mark descriptions for cleanup instead of deleting
-            if to_remove:
-                import time
-
-                cleanup_timestamp = int(time.time())
-                await conn.executemany(
-                    (
-                        "UPDATE file_descriptions SET to_be_cleaned = ? WHERE "
-                        "project_id = ? AND file_path = ?"
-                    ),
-                    [(cleanup_timestamp, project_id, path) for path in to_remove],
-                )
-                logger.info(
-                    (
-                        f"Marked {len(to_remove)} missing files for cleanup "
-                        f"from {project_id}"
-                    )
-                )
+                    missing.append(file_path)
+            return missing
 
-            return to_remove
+        loop = asyncio.get_running_loop()
+        to_remove = await loop.run_in_executor(None, find_removed_files)
 
-        removed_files = await self.execute_transaction_with_retry(
-            cleanup_operation,
-            f"cleanup_missing_files_{project_id}",
-            timeout_seconds=60.0,  # Longer timeout for file system operations
-        )
+        # 3. Mark for cleanup (fast DB write)
+        if to_remove:
+            await self.cleanup_manager.mark_files_for_cleanup(project_id, to_remove)
+            logger.info(
+                f"Marked {len(to_remove)} missing files for cleanup from {project_id}"
+            )
 
-        return removed_files
+        return to_remove
 
     async def analyze_word_frequency(
         self, project_id: str, limit: int = 200
@@ -1139,7 +1173,7 @@ class DatabaseManager:
         stop_words_path = (
             Path(__file__).parent.parent / "data" / "stop_words_english.txt"
         )
-        stop_words = set()
+        stop_words: set = set()
 
         if stop_words_path.exists():
             with open(stop_words_path, "r", encoding="utf-8") as f:
@@ -1186,8 +1220,8 @@ class DatabaseManager:
         }
         stop_words.update(programming_keywords)
 
+        # Get all descriptions for this project (fast DB read)
         async with self.get_connection() as db:
-            # Get all descriptions for this project
             cursor = await db.execute(
                 (
                     "SELECT description FROM file_descriptions WHERE "
@@ -1195,11 +1229,13 @@ class DatabaseManager:
                 ),
                 (project_id,),
             )
-
             rows = await cursor.fetchall()
+            descriptions = [row["description"] for row in rows]
 
+        # Process word frequency in executor (CPU-bound work)
+        def process_word_frequency() -> WordFrequencyResult:
             # Combine all descriptions
-            all_text = " ".join(row["description"] for row in rows)
+            all_text = " ".join(descriptions)
 
             # Tokenize and filter
             words = re.findall(r"\b[a-zA-Z]{2,}\b", all_text.lower())
@@ -1220,6 +1256,9 @@ class DatabaseManager:
                 total_unique_terms=len(word_counts),
             )
 
+        loop = asyncio.get_running_loop()
+        return await loop.run_in_executor(None, process_word_frequency)
+
     async def cleanup_empty_projects(self) -> int:
         """
         Remove projects that have no file descriptions and no project overview.
@@ -1320,6 +1359,186 @@ class DatabaseManager:
                 "files": file_descriptions,
             }
 
+    # IndexMeta operations
+    async def create_index_meta(self, index_meta: IndexMeta) -> None:
+        """Create or update index metadata for a project."""
+        async with self.get_write_connection_with_retry("create_index_meta") as db:
+            await db.execute(
+                """
+                INSERT OR REPLACE INTO index_meta (
+                    project_id, total_chunks, indexed_chunks, total_files, indexed_files,
+                    last_sync, sync_status, error_message, queue_depth, processing_rate,
+                    estimated_completion, metadata, created, last_modified
+                )
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                """,
+                (
+                    index_meta.project_id,
+                    index_meta.total_chunks,
+                    index_meta.indexed_chunks,
+                    index_meta.total_files,
+                    index_meta.indexed_files,
+                    index_meta.last_sync,
+                    index_meta.sync_status.value,
+                    index_meta.error_message,
+                    index_meta.queue_depth,
+                    index_meta.processing_rate,
+                    index_meta.estimated_completion,
+                    json.dumps(index_meta.metadata),
+                    index_meta.created,
+                    index_meta.last_modified,
+                ),
+            )
+            await db.commit()
+            logger.debug(
+                f"Created/updated index metadata for project: {index_meta.project_id}"
+            )
+
+    async def update_index_meta(self, index_meta: IndexMeta) -> None:
+        """Update existing index metadata for a project."""
+        async with self.get_write_connection_with_retry("update_index_meta") as db:
+            await db.execute(
+                """
+                UPDATE index_meta
+                SET total_chunks = ?, indexed_chunks = ?, total_files = ?, indexed_files = ?,
+                    last_sync = ?, sync_status = ?, error_message = ?, queue_depth = ?,
+                    processing_rate = ?, estimated_completion = ?, metadata = ?, last_modified = ?
+                WHERE project_id = ?
+                """,
+                (
+                    index_meta.total_chunks,
+                    index_meta.indexed_chunks,
+                    index_meta.total_files,
+                    index_meta.indexed_files,
+                    index_meta.last_sync,
+                    index_meta.sync_status.value,
+                    index_meta.error_message,
+                    index_meta.queue_depth,
+                    index_meta.processing_rate,
+                    index_meta.estimated_completion,
+                    json.dumps(index_meta.metadata),
+                    index_meta.last_modified,
+                    index_meta.project_id,
+                ),
+            )
+
+            # Check if the project was actually updated
+            cursor = await db.execute("SELECT changes()")
+            changes = await cursor.fetchone()
+            if changes[0] == 0:
+                raise DatabaseError(
+                    f"Index metadata not found for project: {index_meta.project_id}"
+                )
+
+            await db.commit()
+            logger.debug(f"Updated index metadata for project: {index_meta.project_id}")
+
+    async def get_index_meta(self, project_id: str) -> Optional[IndexMeta]:
+        """Retrieve index metadata for a project."""
+        async with self.get_connection() as db:
+            cursor = await db.execute(
+                "SELECT * FROM index_meta WHERE project_id = ?", (project_id,)
+            )
+            row = await cursor.fetchone()
+
+            if row:
+                # Convert row to dict for easier field access
+                row_dict = dict(row)
+
+                # Parse JSON metadata field
+                metadata = (
+                    json.loads(row_dict["metadata"]) if row_dict["metadata"] else {}
+                )
+
+                # Parse datetime fields
+                created = (
+                    datetime.fromisoformat(row_dict["created"])
+                    if row_dict["created"]
+                    else datetime.utcnow()
+                )
+                last_modified = (
+                    datetime.fromisoformat(row_dict["last_modified"])
+                    if row_dict["last_modified"]
+                    else datetime.utcnow()
+                )
+                last_sync = (
+                    datetime.fromisoformat(row_dict["last_sync"])
+                    if row_dict["last_sync"]
+                    else None
+                )
+                estimated_completion = (
+                    datetime.fromisoformat(row_dict["estimated_completion"])
+                    if row_dict["estimated_completion"]
+                    else None
+                )
+
+                return IndexMeta(
+                    id=row_dict["id"],
+                    project_id=row_dict["project_id"],
+                    total_chunks=row_dict["total_chunks"],
+                    indexed_chunks=row_dict["indexed_chunks"],
+                    total_files=row_dict["total_files"],
+                    indexed_files=row_dict["indexed_files"],
+                    last_sync=last_sync,
+                    sync_status=row_dict["sync_status"],
+                    error_message=row_dict["error_message"],
+                    queue_depth=row_dict["queue_depth"],
+                    processing_rate=row_dict["processing_rate"],
+                    estimated_completion=estimated_completion,
+                    metadata=metadata,
+                    created=created,
+                    last_modified=last_modified,
+                )
+            return None
+
+    async def get_or_create_index_meta(self, project_id: str, **kwargs) -> IndexMeta:
+        """
+        Get existing index metadata or create new one with default values.
+        
+        Args:
+            project_id: Project identifier
+            **kwargs: Optional fields to override defaults when creating new metadata
+            
+        Returns:
+            IndexMeta object (existing or newly created)
+        """
+        # Try to get existing metadata first
+        existing_meta = await self.get_index_meta(project_id)
+        if existing_meta:
+            return existing_meta
+        
+        # Create new metadata with defaults, allowing kwargs to override
+        default_metadata = {
+            "project_id": project_id,
+            "total_chunks": 0,
+            "indexed_chunks": 0,
+            "total_files": 0,
+            "indexed_files": 0,
+            "last_sync": None,
+            "sync_status": SyncStatus.PENDING,
+            "error_message": None,
+            "queue_depth": 0,
+            "processing_rate": 0.0,
+            "estimated_completion": None,
+            "metadata": {},
+        }
+        
+        # Override defaults with provided kwargs
+        default_metadata.update(kwargs)
+        
+        # Create the IndexMeta object
+        new_meta = IndexMeta(**default_metadata)
+        
+        # Store it in the database
+        await self.create_index_meta(new_meta)
+        
+        # Return the created metadata (fetch it back to get the assigned ID)
+        result = await self.get_index_meta(project_id)
+        if result is None:
+            raise DatabaseError(f"Failed to create index metadata for project: {project_id}")
+        
+        return result
+
     # Cleanup operations
 
     @property

mcp_code_indexer/database/database_factory.py

@@ -28,7 +28,7 @@ class DatabaseFactory:
         global_db_path: Path,
         pool_size: int = 3,
         retry_count: int = 5,
-        timeout: float = 10.0,
+        timeout: float = 30.0,
         enable_wal_mode: bool = True,
         health_check_interval: float = 30.0,
         retry_min_wait: float = 0.1,

mcp_code_indexer/database/exceptions.py

@@ -236,7 +236,7 @@ def classify_sqlite_error(error: Exception, operation_name: str = "") -> Databas
         for msg in [
             "no such table",
             "no such column",
-            "table already exists",
+            "already exists",
             "syntax error",
         ]
     ):