mcp-code-indexer 4.2.14__py3-none-any.whl → 4.2.16__py3-none-any.whl

mcp_code_indexer/database/database.py

@@ -27,9 +27,11 @@ from mcp_code_indexer.database.exceptions import (
 )
 from mcp_code_indexer.database.models import (
     FileDescription,
+    IndexMeta,
     Project,
     ProjectOverview,
     SearchResult,
+    SyncStatus,
     WordFrequencyResult,
     WordFrequencyTerm,
 )
@@ -315,12 +317,10 @@ class DatabaseManager:
         self, operation_name: str = "write_operation"
     ) -> AsyncIterator[aiosqlite.Connection]:
         """
-        Get a database connection with write serialization and automatic
-        retry logic.
+        Get a database connection with write serialization.
 
-        This uses the new RetryExecutor to properly handle retry logic
-        without the broken yield-in-retry-loop pattern that caused
-        generator errors.
+        Ensures the write lock is held throughout the duration of the context
+        to prevent race conditions and database locking errors.
 
         Args:
             operation_name: Name of the operation for logging and
@@ -331,43 +331,10 @@ class DatabaseManager:
                 "DatabaseManager not initialized - call initialize() first"
             )
 
-        async def get_write_connection() -> aiosqlite.Connection:
-            """Inner function to get connection - retried by executor."""
-            if self._write_lock is None:
-                raise RuntimeError("Write lock not initialized")
-            async with self._write_lock:
-                async with self.get_connection() as conn:
-                    return conn
-
-        try:
-            # Use retry executor to handle connection acquisition with retries
-            connection = await self._retry_executor.execute_with_retry(
-                get_write_connection, operation_name
-            )
-
-            try:
-                yield connection
-
-                # Success - retry executor handles all failure tracking
-
-            except Exception:
-                # Error handling is managed by the retry executor
-                raise
-
-        except DatabaseError:
-            # Re-raise our custom database errors as-is
-            raise
-        except Exception as e:
-            # Classify and wrap other exceptions
-            classified_error = classify_sqlite_error(e, operation_name)
-            logger.error(
-                (
-                    f"Database operation '{operation_name}' failed: "
-                    f"{classified_error.message}"
-                ),
-                extra={"structured_data": classified_error.to_dict()},
-            )
-            raise classified_error
+        # Acquire lock for exclusive write access - hold it for entire context
+        async with self._write_lock:
+            async with self.get_connection() as conn:
+                yield conn
 
     def get_database_stats(self) -> Dict[str, Any]:
         """
@@ -740,6 +707,25 @@ class DatabaseManager:
             await db.commit()
             logger.debug(f"Updated project: {project.id}")
 
+    async def set_project_vector_mode(self, project_id: str, enabled: bool) -> None:
+        """Set the vector_mode for a specific project."""
+        async with self.get_write_connection_with_retry(
+            "set_project_vector_mode"
+        ) as db:
+            await db.execute(
+                "UPDATE projects SET vector_mode = ? WHERE id = ?",
+                (int(enabled), project_id),
+            )
+
+            # Check if the project was actually updated
+            cursor = await db.execute("SELECT changes()")
+            changes = await cursor.fetchone()
+            if changes[0] == 0:
+                raise ValueError(f"Project not found: {project_id}")
+
+            await db.commit()
+            logger.debug(f"Set vector_mode={enabled} for project: {project_id}")
+
     async def get_all_projects(self) -> List[Project]:
         """Get all projects in the database."""
         async with self.get_connection() as db:
@@ -795,12 +781,19 @@ class DatabaseManager:
         ) as db:
             await db.execute(
                 """
-                INSERT OR REPLACE INTO file_descriptions
+                INSERT INTO file_descriptions
                 (
                     project_id, file_path, description, file_hash, last_modified,
                     version, source_project_id, to_be_cleaned
                 )
                 VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+                ON CONFLICT(project_id, file_path) DO UPDATE SET
+                    description=excluded.description,
+                    file_hash=excluded.file_hash,
+                    last_modified=excluded.last_modified,
+                    version=excluded.version,
+                    source_project_id=excluded.source_project_id,
+                    to_be_cleaned=excluded.to_be_cleaned
                 """,
                 (
                     file_desc.project_id,
@@ -898,12 +891,19 @@ class DatabaseManager:
 
             await conn.executemany(
                 """
-                INSERT OR REPLACE INTO file_descriptions
+                INSERT INTO file_descriptions
                 (
                     project_id, file_path, description, file_hash, last_modified,
                     version, source_project_id, to_be_cleaned
                 )
                 VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+                ON CONFLICT(project_id, file_path) DO UPDATE SET
+                    description=excluded.description,
+                    file_hash=excluded.file_hash,
+                    last_modified=excluded.last_modified,
+                    version=excluded.version,
+                    source_project_id=excluded.source_project_id,
+                    to_be_cleaned=excluded.to_be_cleaned
                 """,
                 data,
             )
@@ -1067,10 +1067,8 @@ class DatabaseManager:
         Returns:
             List of file paths that were marked for cleanup
         """
-        removed_files: List[str] = []
-
-        async def cleanup_operation(conn: aiosqlite.Connection) -> List[str]:
-            # Get all active file descriptions for this project
+        # 1. Get all active file paths (fast DB read)
+        async with self.get_connection() as conn:
             cursor = await conn.execute(
                 (
                     "SELECT file_path FROM file_descriptions WHERE "
@@ -1078,46 +1076,29 @@ class DatabaseManager:
                 ),
                 (project_id,),
             )
-
             rows = await cursor.fetchall()
+            file_paths = [row["file_path"] for row in rows]
 
-            # Check which files no longer exist
-            to_remove = []
-            for row in rows:
-                file_path = row["file_path"]
+        # 2. Check existence on disk (blocking IO - run in executor)
+        def find_removed_files() -> List[str]:
+            missing = []
+            for file_path in file_paths:
                 full_path = project_root / file_path
-
                 if not full_path.exists():
-                    to_remove.append(file_path)
-
-            # Mark descriptions for cleanup instead of deleting
-            if to_remove:
-                import time
-
-                cleanup_timestamp = int(time.time())
-                await conn.executemany(
-                    (
-                        "UPDATE file_descriptions SET to_be_cleaned = ? WHERE "
-                        "project_id = ? AND file_path = ?"
-                    ),
-                    [(cleanup_timestamp, project_id, path) for path in to_remove],
-                )
-                logger.info(
-                    (
-                        f"Marked {len(to_remove)} missing files for cleanup "
-                        f"from {project_id}"
-                    )
-                )
+                    missing.append(file_path)
+            return missing
 
-            return to_remove
+        loop = asyncio.get_running_loop()
+        to_remove = await loop.run_in_executor(None, find_removed_files)
 
-        removed_files = await self.execute_transaction_with_retry(
-            cleanup_operation,
-            f"cleanup_missing_files_{project_id}",
-            timeout_seconds=60.0,  # Longer timeout for file system operations
-        )
+        # 3. Mark for cleanup (fast DB write)
+        if to_remove:
+            await self.cleanup_manager.mark_files_for_cleanup(project_id, to_remove)
+            logger.info(
+                f"Marked {len(to_remove)} missing files for cleanup from {project_id}"
+            )
 
-        return removed_files
+        return to_remove
 
     async def analyze_word_frequency(
         self, project_id: str, limit: int = 200
@@ -1139,7 +1120,7 @@ class DatabaseManager:
         stop_words_path = (
             Path(__file__).parent.parent / "data" / "stop_words_english.txt"
         )
-        stop_words = set()
+        stop_words: set = set()
 
         if stop_words_path.exists():
             with open(stop_words_path, "r", encoding="utf-8") as f:
@@ -1186,8 +1167,8 @@ class DatabaseManager:
         }
         stop_words.update(programming_keywords)
 
+        # Get all descriptions for this project (fast DB read)
         async with self.get_connection() as db:
-            # Get all descriptions for this project
             cursor = await db.execute(
                 (
                     "SELECT description FROM file_descriptions WHERE "
@@ -1195,11 +1176,13 @@ class DatabaseManager:
                 ),
                 (project_id,),
             )
-
             rows = await cursor.fetchall()
+            descriptions = [row["description"] for row in rows]
 
+        # Process word frequency in executor (CPU-bound work)
+        def process_word_frequency() -> WordFrequencyResult:
             # Combine all descriptions
-            all_text = " ".join(row["description"] for row in rows)
+            all_text = " ".join(descriptions)
 
             # Tokenize and filter
             words = re.findall(r"\b[a-zA-Z]{2,}\b", all_text.lower())
@@ -1220,6 +1203,9 @@ class DatabaseManager:
                 total_unique_terms=len(word_counts),
             )
 
+        loop = asyncio.get_running_loop()
+        return await loop.run_in_executor(None, process_word_frequency)
+
     async def cleanup_empty_projects(self) -> int:
         """
         Remove projects that have no file descriptions and no project overview.
@@ -1320,6 +1306,186 @@ class DatabaseManager:
                 "files": file_descriptions,
             }
 
+    # IndexMeta operations
+    async def create_index_meta(self, index_meta: IndexMeta) -> None:
+        """Create or update index metadata for a project."""
+        async with self.get_write_connection_with_retry("create_index_meta") as db:
+            await db.execute(
+                """
+                INSERT OR REPLACE INTO index_meta (
+                    project_id, total_chunks, indexed_chunks, total_files, indexed_files,
+                    last_sync, sync_status, error_message, queue_depth, processing_rate,
+                    estimated_completion, metadata, created, last_modified
+                )
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                """,
+                (
+                    index_meta.project_id,
+                    index_meta.total_chunks,
+                    index_meta.indexed_chunks,
+                    index_meta.total_files,
+                    index_meta.indexed_files,
+                    index_meta.last_sync,
+                    index_meta.sync_status.value,
+                    index_meta.error_message,
+                    index_meta.queue_depth,
+                    index_meta.processing_rate,
+                    index_meta.estimated_completion,
+                    json.dumps(index_meta.metadata),
+                    index_meta.created,
+                    index_meta.last_modified,
+                ),
+            )
+            await db.commit()
+            logger.debug(
+                f"Created/updated index metadata for project: {index_meta.project_id}"
+            )
+
+    async def update_index_meta(self, index_meta: IndexMeta) -> None:
+        """Update existing index metadata for a project."""
+        async with self.get_write_connection_with_retry("update_index_meta") as db:
+            await db.execute(
+                """
+                UPDATE index_meta
+                SET total_chunks = ?, indexed_chunks = ?, total_files = ?, indexed_files = ?,
+                    last_sync = ?, sync_status = ?, error_message = ?, queue_depth = ?,
+                    processing_rate = ?, estimated_completion = ?, metadata = ?, last_modified = ?
+                WHERE project_id = ?
+                """,
+                (
+                    index_meta.total_chunks,
+                    index_meta.indexed_chunks,
+                    index_meta.total_files,
+                    index_meta.indexed_files,
+                    index_meta.last_sync,
+                    index_meta.sync_status.value,
+                    index_meta.error_message,
+                    index_meta.queue_depth,
+                    index_meta.processing_rate,
+                    index_meta.estimated_completion,
+                    json.dumps(index_meta.metadata),
+                    index_meta.last_modified,
+                    index_meta.project_id,
+                ),
+            )
+
+            # Check if the project was actually updated
+            cursor = await db.execute("SELECT changes()")
+            changes = await cursor.fetchone()
+            if changes[0] == 0:
+                raise ValueError(
+                    f"Index metadata not found for project: {index_meta.project_id}"
+                )
+
+            await db.commit()
+            logger.debug(f"Updated index metadata for project: {index_meta.project_id}")
+
+    async def get_index_meta(self, project_id: str) -> Optional[IndexMeta]:
+        """Retrieve index metadata for a project."""
+        async with self.get_connection() as db:
+            cursor = await db.execute(
+                "SELECT * FROM index_meta WHERE project_id = ?", (project_id,)
+            )
+            row = await cursor.fetchone()
+
+            if row:
+                # Convert row to dict for easier field access
+                row_dict = dict(row)
+
+                # Parse JSON metadata field
+                metadata = (
+                    json.loads(row_dict["metadata"]) if row_dict["metadata"] else {}
+                )
+
+                # Parse datetime fields
+                created = (
+                    datetime.fromisoformat(row_dict["created"])
+                    if row_dict["created"]
+                    else datetime.utcnow()
+                )
+                last_modified = (
+                    datetime.fromisoformat(row_dict["last_modified"])
+                    if row_dict["last_modified"]
+                    else datetime.utcnow()
+                )
+                last_sync = (
+                    datetime.fromisoformat(row_dict["last_sync"])
+                    if row_dict["last_sync"]
+                    else None
+                )
+                estimated_completion = (
+                    datetime.fromisoformat(row_dict["estimated_completion"])
+                    if row_dict["estimated_completion"]
+                    else None
+                )
+
+                return IndexMeta(
+                    id=row_dict["id"],
+                    project_id=row_dict["project_id"],
+                    total_chunks=row_dict["total_chunks"],
+                    indexed_chunks=row_dict["indexed_chunks"],
+                    total_files=row_dict["total_files"],
+                    indexed_files=row_dict["indexed_files"],
+                    last_sync=last_sync,
+                    sync_status=row_dict["sync_status"],
+                    error_message=row_dict["error_message"],
+                    queue_depth=row_dict["queue_depth"],
+                    processing_rate=row_dict["processing_rate"],
+                    estimated_completion=estimated_completion,
+                    metadata=metadata,
+                    created=created,
+                    last_modified=last_modified,
+                )
+            return None
+
+    async def get_or_create_index_meta(self, project_id: str, **kwargs) -> IndexMeta:
+        """
+        Get existing index metadata or create new one with default values.
+        
+        Args:
+            project_id: Project identifier
+            **kwargs: Optional fields to override defaults when creating new metadata
+            
+        Returns:
+            IndexMeta object (existing or newly created)
+        """
+        # Try to get existing metadata first
+        existing_meta = await self.get_index_meta(project_id)
+        if existing_meta:
+            return existing_meta
+        
+        # Create new metadata with defaults, allowing kwargs to override
+        default_metadata = {
+            "project_id": project_id,
+            "total_chunks": 0,
+            "indexed_chunks": 0,
+            "total_files": 0,
+            "indexed_files": 0,
+            "last_sync": None,
+            "sync_status": SyncStatus.PENDING,
+            "error_message": None,
+            "queue_depth": 0,
+            "processing_rate": 0.0,
+            "estimated_completion": None,
+            "metadata": {},
+        }
+        
+        # Override defaults with provided kwargs
+        default_metadata.update(kwargs)
+        
+        # Create the IndexMeta object
+        new_meta = IndexMeta(**default_metadata)
+        
+        # Store it in the database
+        await self.create_index_meta(new_meta)
+        
+        # Return the created metadata (fetch it back to get the assigned ID)
+        result = await self.get_index_meta(project_id)
+        if result is None:
+            raise DatabaseError(f"Failed to create index metadata for project: {project_id}")
+        
+        return result
+
     # Cleanup operations
 
     @property

mcp_code_indexer/database/models.py

@@ -32,7 +32,9 @@ class Project(BaseModel):
     last_accessed: datetime = Field(
         default_factory=datetime.utcnow, description="Last access timestamp"
     )
-    vector_mode: bool = Field(default=False, description="Enable vector search for this project")
+    vector_mode: bool = Field(
+        default=False, description="Enable vector search for this project"
+    )
 
 
 class FileDescription(BaseModel):
@@ -189,10 +191,12 @@ class WordFrequencyResult(BaseModel):
 
 # Vector Mode Models
 
+
 class ChunkType(str, Enum):
     """Types of code chunks for semantic analysis."""
+
     FUNCTION = "function"
-    CLASS = "class"  
+    CLASS = "class"
     METHOD = "method"
     IMPORT = "import"
     DOCSTRING = "docstring"
@@ -204,27 +208,32 @@ class ChunkType(str, Enum):
     NAMESPACE = "namespace"
     GENERIC = "generic"
 
+
 class NodeType(str, Enum):
     """Types of nodes in Merkle tree."""
+
     FILE = "file"
     DIRECTORY = "directory"
     PROJECT = "project"
 
+
 class SyncStatus(str, Enum):
     """Vector index synchronization status."""
+
     PENDING = "pending"
     IN_PROGRESS = "in_progress"
     COMPLETED = "completed"
     FAILED = "failed"
     PAUSED = "paused"
 
+
 class CodeChunk(BaseModel):
     """
     Represents a semantic chunk of code extracted from a file.
-    
+
     Used for embedding generation and vector search operations.
     """
-    
+
     id: Optional[int] = Field(None, description="Database ID")
     file_id: int = Field(..., description="Reference to FileDescription")
     project_id: str = Field(..., description="Reference to project")
@@ -235,17 +244,24 @@ class CodeChunk(BaseModel):
     content_hash: str = Field(..., description="SHA-256 hash of chunk content")
     embedding_id: Optional[str] = Field(None, description="Vector database ID")
     redacted: bool = Field(default=False, description="Whether content was redacted")
-    metadata: Dict[str, Any] = Field(default_factory=dict, description="Additional metadata")
-    created: datetime = Field(default_factory=datetime.utcnow, description="Creation timestamp")
-    last_modified: datetime = Field(default_factory=datetime.utcnow, description="Last update timestamp")
+    metadata: Dict[str, Any] = Field(
+        default_factory=dict, description="Additional metadata"
+    )
+    created: datetime = Field(
+        default_factory=datetime.utcnow, description="Creation timestamp"
+    )
+    last_modified: datetime = Field(
+        default_factory=datetime.utcnow, description="Last update timestamp"
+    )
+
 
 class MerkleNode(BaseModel):
     """
     Represents a node in the Merkle tree for change detection.
-    
+
     Used to efficiently detect file system changes without scanning entire directory trees.
     """
-    
+
     id: Optional[int] = Field(None, description="Database ID")
     project_id: str = Field(..., description="Reference to project")
     path: str = Field(..., description="File/directory path relative to project root")
@@ -253,36 +269,56 @@ class MerkleNode(BaseModel):
     node_type: NodeType = Field(..., description="Type of filesystem node")
     parent_path: Optional[str] = Field(None, description="Path to parent directory")
     children_hash: Optional[str] = Field(None, description="Combined hash of children")
-    last_modified: datetime = Field(default_factory=datetime.utcnow, description="Last update timestamp")
+    last_modified: datetime = Field(
+        default_factory=datetime.utcnow, description="Last update timestamp"
+    )
+
 
 class IndexMeta(BaseModel):
     """
     Metadata about vector indexing progress and status for a project.
-    
+
     Tracks indexing state, statistics, and synchronization status.
     """
-    
+
     id: Optional[int] = Field(None, description="Database ID")
     project_id: str = Field(..., description="Reference to project", unique=True)
     total_chunks: int = Field(default=0, description="Total number of chunks")
-    indexed_chunks: int = Field(default=0, description="Number of chunks with embeddings")
+    indexed_chunks: int = Field(
+        default=0, description="Number of chunks with embeddings"
+    )
     total_files: int = Field(default=0, description="Total number of files")
     indexed_files: int = Field(default=0, description="Number of files processed")
-    last_sync: Optional[datetime] = Field(None, description="Last successful sync timestamp")
-    sync_status: SyncStatus = Field(default=SyncStatus.PENDING, description="Current sync status")
+    last_sync: Optional[datetime] = Field(
+        None, description="Last successful sync timestamp"
+    )
+    sync_status: SyncStatus = Field(
+        default=SyncStatus.PENDING, description="Current sync status"
+    )
     error_message: Optional[str] = Field(None, description="Last error message")
     queue_depth: int = Field(default=0, description="Number of pending tasks")
-    processing_rate: float = Field(default=0.0, description="Files per second processing rate")
-    estimated_completion: Optional[datetime] = Field(None, description="Estimated completion time")
-    metadata: Dict[str, Any] = Field(default_factory=dict, description="Additional metadata")
-    created: datetime = Field(default_factory=datetime.utcnow, description="Creation timestamp")
-    last_modified: datetime = Field(default_factory=datetime.utcnow, description="Last update timestamp")
+    processing_rate: float = Field(
+        default=0.0, description="Files per second processing rate"
+    )
+    estimated_completion: Optional[datetime] = Field(
+        None, description="Estimated completion time"
+    )
+    metadata: Dict[str, Any] = Field(
+        default_factory=dict, description="Additional metadata"
+    )
+    created: datetime = Field(
+        default_factory=datetime.utcnow, description="Creation timestamp"
+    )
+    last_modified: datetime = Field(
+        default_factory=datetime.utcnow, description="Last update timestamp"
+    )
+
 
 class VectorSearchResult(BaseModel):
     """
     Represents a vector search result with similarity scoring.
     """
-    
+
     file_path: str = Field(..., description="Path to the matching file")
     chunk_name: Optional[str] = Field(None, description="Name of the code chunk")
     chunk_type: ChunkType = Field(..., description="Type of code chunk")
@@ -291,13 +327,16 @@ class VectorSearchResult(BaseModel):
     end_line: int = Field(..., description="Ending line number")
     similarity_score: float = Field(..., description="Cosine similarity score")
     project_id: str = Field(..., description="Project identifier")
-    metadata: Dict[str, Any] = Field(default_factory=dict, description="Additional metadata")
+    metadata: Dict[str, Any] = Field(
+        default_factory=dict, description="Additional metadata"
+    )
+
 
 class VectorIndexStatus(BaseModel):
     """
     Current status of vector indexing for a project.
     """
-    
+
     is_indexing: bool = Field(..., description="Whether indexing is currently active")
     indexed_files: int = Field(..., description="Number of files indexed")
     total_files: int = Field(..., description="Total number of files")
@@ -307,9 +346,12 @@ class VectorIndexStatus(BaseModel):
     sync_status: SyncStatus = Field(..., description="Current sync status")
     queue_depth: int = Field(..., description="Number of pending tasks")
     processing_rate: float = Field(..., description="Processing rate")
-    estimated_completion: Optional[datetime] = Field(None, description="Estimated completion time")
+    estimated_completion: Optional[datetime] = Field(
+        None, description="Estimated completion time"
+    )
     error_message: Optional[str] = Field(None, description="Last error message")
 
+
 # Enable forward references for recursive models
 FolderNode.model_rebuild()
 CodebaseOverview.model_rebuild()

mcp_code_indexer/database/retry_executor.py

@@ -279,8 +279,13 @@ class RetryExecutor:
         Yields:
             Database connection
         """
+        import sys
+
+        # Store the context manager so we can properly call __aexit__
+        ctx_manager: Optional[AsyncContextManager[aiosqlite.Connection]] = None
 
         async def acquire_connection() -> aiosqlite.Connection:
+            nonlocal ctx_manager
             # This function will be retried by execute_with_retry
             # Get the async context manager and enter it
             ctx_manager = connection_factory()
@@ -288,15 +293,20 @@ class RetryExecutor:
             return conn
 
         # Use execute_with_retry to handle the retry logic
-        # We create a connection and store it for the context manager
         connection = await self.execute_with_retry(acquire_connection, operation_name)
 
         try:
             yield connection
-        finally:
-            # Close the connection properly
-            if hasattr(connection, "close"):
-                await connection.close()
+        except BaseException:
+            # Pass actual exception info to __aexit__ for proper rollback/cleanup
+            exc_type, exc, tb = sys.exc_info()
+            if ctx_manager is not None:
+                await ctx_manager.__aexit__(exc_type, exc, tb)
+            raise
+        else:
+            # No exception - call __aexit__ with None values
+            if ctx_manager is not None:
+                await ctx_manager.__aexit__(None, None, None)
 
     def _should_retry_exception(self, retry_state: RetryCallState) -> bool:
         """