mcp-vector-search 0.0.3__py3-none-any.whl

mcp_vector_search/core/watcher.py

@@ -0,0 +1,320 @@
+"""File system watcher for incremental indexing."""
+
+import asyncio
+from pathlib import Path
+from typing import List, Optional, Set, Callable, Awaitable, Union
+from threading import Thread
+import time
+from concurrent.futures import Future
+
+from loguru import logger
+from watchdog.observers import Observer
+from watchdog.events import FileSystemEventHandler, FileSystemEvent
+
+from ..config.settings import ProjectConfig
+from .indexer import SemanticIndexer
+from .database import ChromaVectorDatabase
+
+
+class CodeFileHandler(FileSystemEventHandler):
+    """Handler for code file changes."""
+
+    def __init__(
+        self,
+        file_extensions: List[str],
+        ignore_patterns: List[str],
+        callback: Callable[[str, str], Awaitable[None]],
+        loop: asyncio.AbstractEventLoop,
+        debounce_delay: float = 1.0,
+    ):
+        """Initialize file handler.
+
+        Args:
+            file_extensions: List of file extensions to watch
+            ignore_patterns: List of patterns to ignore
+            callback: Async callback function for file changes
+            loop: Event loop to schedule tasks on
+            debounce_delay: Delay in seconds to debounce rapid changes
+        """
+        super().__init__()
+        self.file_extensions = set(file_extensions)
+        self.ignore_patterns = ignore_patterns
+        self.callback = callback
+        self.loop = loop
+        self.debounce_delay = debounce_delay
+        self.pending_changes: Set[str] = set()
+        self.last_change_time: float = 0
+        self.debounce_task: Optional[Union[asyncio.Task, Future]] = None
+
+    def should_process_file(self, file_path: str) -> bool:
+        """Check if file should be processed."""
+        path = Path(file_path)
+        
+        # Check file extension
+        if path.suffix not in self.file_extensions:
+            return False
+        
+        # Check ignore patterns
+        for pattern in self.ignore_patterns:
+            if pattern in str(path):
+                return False
+        
+        return True
+
+    def on_modified(self, event: FileSystemEvent) -> None:
+        """Handle file modification."""
+        if not event.is_directory and self.should_process_file(event.src_path):
+            self._schedule_change(event.src_path, "modified")
+
+    def on_created(self, event: FileSystemEvent) -> None:
+        """Handle file creation."""
+        if not event.is_directory and self.should_process_file(event.src_path):
+            self._schedule_change(event.src_path, "created")
+
+    def on_deleted(self, event: FileSystemEvent) -> None:
+        """Handle file deletion."""
+        if not event.is_directory and self.should_process_file(event.src_path):
+            self._schedule_change(event.src_path, "deleted")
+
+    def on_moved(self, event: FileSystemEvent) -> None:
+        """Handle file move/rename."""
+        if hasattr(event, 'dest_path'):
+            # Handle rename/move
+            if not event.is_directory:
+                if self.should_process_file(event.src_path):
+                    self._schedule_change(event.src_path, "deleted")
+                if self.should_process_file(event.dest_path):
+                    self._schedule_change(event.dest_path, "created")
+
+    def _schedule_change(self, file_path: str, change_type: str) -> None:
+        """Schedule a file change for processing with debouncing."""
+        self.pending_changes.add(f"{change_type}:{file_path}")
+        self.last_change_time = time.time()
+
+        # Cancel existing debounce task
+        if self.debounce_task and not self.debounce_task.done():
+            self.debounce_task.cancel()
+
+        # Schedule new debounce task using the stored loop
+        future = asyncio.run_coroutine_threadsafe(
+            self._debounced_process(), self.loop
+        )
+        # Store the future as our task (it has a done() method)
+        self.debounce_task = future
+
+    async def _debounced_process(self) -> None:
+        """Process pending changes after debounce delay."""
+        await asyncio.sleep(self.debounce_delay)
+        
+        # Check if more changes occurred during debounce
+        if time.time() - self.last_change_time < self.debounce_delay:
+            return
+        
+        # Process all pending changes
+        changes = self.pending_changes.copy()
+        self.pending_changes.clear()
+        
+        for change in changes:
+            change_type, file_path = change.split(":", 1)
+            try:
+                await self.callback(file_path, change_type)
+            except Exception as e:
+                logger.error(f"Error processing file change {file_path}: {e}")
+
+
+class FileWatcher:
+    """File system watcher for incremental indexing."""
+
+    def __init__(
+        self,
+        project_root: Path,
+        config: ProjectConfig,
+        indexer: SemanticIndexer,
+        database: ChromaVectorDatabase,
+    ):
+        """Initialize file watcher.
+        
+        Args:
+            project_root: Root directory to watch
+            config: Project configuration
+            indexer: Semantic indexer instance
+            database: Vector database instance
+        """
+        self.project_root = project_root
+        self.config = config
+        self.indexer = indexer
+        self.database = database
+        self.observer: Optional[Observer] = None
+        self.handler: Optional[CodeFileHandler] = None
+        self.is_running = False
+
+    async def start(self) -> None:
+        """Start watching for file changes."""
+        if self.is_running:
+            logger.warning("File watcher is already running")
+            return
+
+        logger.info(f"Starting file watcher for {self.project_root}")
+        
+        # Create handler
+        loop = asyncio.get_running_loop()
+        self.handler = CodeFileHandler(
+            file_extensions=self.config.file_extensions,
+            ignore_patterns=self._get_ignore_patterns(),
+            callback=self._handle_file_change,
+            loop=loop,
+            debounce_delay=1.0,
+        )
+
+        # Create observer
+        self.observer = Observer()
+        self.observer.schedule(
+            self.handler,
+            str(self.project_root),
+            recursive=True
+        )
+
+        # Start observer in a separate thread
+        self.observer.start()
+        self.is_running = True
+        
+        logger.info("File watcher started successfully")
+
+    async def stop(self) -> None:
+        """Stop watching for file changes."""
+        if not self.is_running:
+            return
+
+        logger.info("Stopping file watcher")
+        
+        if self.observer:
+            self.observer.stop()
+            self.observer.join()
+            self.observer = None
+
+        self.handler = None
+        self.is_running = False
+        
+        logger.info("File watcher stopped")
+
+    def _get_ignore_patterns(self) -> List[str]:
+        """Get patterns to ignore during watching."""
+        default_patterns = [
+            ".git", ".svn", ".hg",
+            "__pycache__", ".pytest_cache",
+            "node_modules", ".venv", "venv",
+            ".DS_Store", "Thumbs.db",
+            ".idea", ".vscode",
+            "build", "dist", "target",
+            ".mcp-vector-search",  # Ignore our own index directory
+        ]
+        
+        # Add any custom ignore patterns from config
+        # TODO: Add custom ignore patterns to config
+        return default_patterns
+
+    async def _handle_file_change(self, file_path: str, change_type: str) -> None:
+        """Handle a file change event.
+        
+        Args:
+            file_path: Path to the changed file
+            change_type: Type of change (created, modified, deleted)
+        """
+        path = Path(file_path)
+        logger.debug(f"Processing file change: {change_type} {path}")
+
+        try:
+            if change_type == "deleted":
+                # Remove chunks for deleted file
+                await self._remove_file_chunks(path)
+            elif change_type in ("created", "modified"):
+                # Re-index the file
+                await self._reindex_file(path)
+            
+            logger.info(f"Processed {change_type} for {path.name}")
+            
+        except Exception as e:
+            logger.error(f"Failed to process {change_type} for {path}: {e}")
+
+    async def _remove_file_chunks(self, file_path: Path) -> None:
+        """Remove all chunks for a deleted file."""
+        # Get relative path for consistent IDs
+        try:
+            relative_path = file_path.relative_to(self.project_root)
+        except ValueError:
+            relative_path = file_path
+
+        # Remove chunks from database
+        await self.database.remove_file_chunks(str(relative_path))
+        logger.debug(f"Removed chunks for deleted file: {relative_path}")
+
+    async def _reindex_file(self, file_path: Path) -> None:
+        """Re-index a single file."""
+        if not file_path.exists():
+            logger.warning(f"File no longer exists: {file_path}")
+            return
+
+        # Remove existing chunks first
+        await self._remove_file_chunks(file_path)
+        
+        # Index the file
+        chunks_indexed = await self.indexer.index_file(file_path)
+        logger.debug(f"Re-indexed {file_path.name}: {chunks_indexed} chunks")
+
+    async def __aenter__(self):
+        """Async context manager entry."""
+        await self.start()
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        """Async context manager exit."""
+        await self.stop()
+
+
+class WatcherManager:
+    """Manager for file watchers across multiple projects."""
+
+    def __init__(self):
+        """Initialize watcher manager."""
+        self.watchers: dict[str, FileWatcher] = {}
+
+    async def start_watcher(
+        self,
+        project_root: Path,
+        config: ProjectConfig,
+        indexer: SemanticIndexer,
+        database: ChromaVectorDatabase,
+    ) -> FileWatcher:
+        """Start a file watcher for a project."""
+        project_key = str(project_root)
+        
+        if project_key in self.watchers:
+            logger.warning(f"Watcher already exists for {project_root}")
+            return self.watchers[project_key]
+
+        watcher = FileWatcher(project_root, config, indexer, database)
+        await watcher.start()
+        
+        self.watchers[project_key] = watcher
+        return watcher
+
+    async def stop_watcher(self, project_root: Path) -> None:
+        """Stop a file watcher for a project."""
+        project_key = str(project_root)
+        
+        if project_key not in self.watchers:
+            logger.warning(f"No watcher found for {project_root}")
+            return
+
+        watcher = self.watchers.pop(project_key)
+        await watcher.stop()
+
+    async def stop_all(self) -> None:
+        """Stop all file watchers."""
+        for watcher in list(self.watchers.values()):
+            await watcher.stop()
+        self.watchers.clear()
+
+    def is_watching(self, project_root: Path) -> bool:
+        """Check if a project is being watched."""
+        return str(project_root) in self.watchers

mcp_vector_search/mcp/__init__.py

@@ -0,0 +1 @@
+"""MCP server integration for MCP Vector Search."""

mcp_vector_search/parsers/__init__.py

@@ -0,0 +1 @@
+"""Language parsers for MCP Vector Search."""

mcp_vector_search/parsers/base.py

@@ -0,0 +1,180 @@
+"""Base parser interface for MCP Vector Search."""
+
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import List, Optional
+
+from ..core.models import CodeChunk
+
+
+class BaseParser(ABC):
+    """Abstract base class for language parsers."""
+
+    def __init__(self, language: str) -> None:
+        """Initialize parser for a specific language.
+        
+        Args:
+            language: Programming language name
+        """
+        self.language = language
+
+    @abstractmethod
+    async def parse_file(self, file_path: Path) -> List[CodeChunk]:
+        """Parse a file and extract code chunks.
+        
+        Args:
+            file_path: Path to the file to parse
+            
+        Returns:
+            List of code chunks extracted from the file
+        """
+        ...
+
+    @abstractmethod
+    async def parse_content(self, content: str, file_path: Path) -> List[CodeChunk]:
+        """Parse content and extract code chunks.
+        
+        Args:
+            content: File content to parse
+            file_path: Path to the source file (for metadata)
+            
+        Returns:
+            List of code chunks extracted from the content
+        """
+        ...
+
+    def supports_file(self, file_path: Path) -> bool:
+        """Check if this parser supports the given file.
+        
+        Args:
+            file_path: Path to check
+            
+        Returns:
+            True if this parser can handle the file
+        """
+        return file_path.suffix.lower() in self.get_supported_extensions()
+
+    @abstractmethod
+    def get_supported_extensions(self) -> List[str]:
+        """Get list of file extensions supported by this parser.
+        
+        Returns:
+            List of file extensions (including the dot)
+        """
+        ...
+
+    def _create_chunk(
+        self,
+        content: str,
+        file_path: Path,
+        start_line: int,
+        end_line: int,
+        chunk_type: str = "code",
+        function_name: Optional[str] = None,
+        class_name: Optional[str] = None,
+        docstring: Optional[str] = None,
+    ) -> CodeChunk:
+        """Create a code chunk with metadata.
+        
+        Args:
+            content: Code content
+            file_path: Source file path
+            start_line: Starting line number (1-based)
+            end_line: Ending line number (1-based)
+            chunk_type: Type of chunk (code, function, class, etc.)
+            function_name: Function name if applicable
+            class_name: Class name if applicable
+            docstring: Docstring if applicable
+            
+        Returns:
+            CodeChunk instance
+        """
+        return CodeChunk(
+            content=content.strip(),
+            file_path=file_path,
+            start_line=start_line,
+            end_line=end_line,
+            language=self.language,
+            chunk_type=chunk_type,
+            function_name=function_name,
+            class_name=class_name,
+            docstring=docstring,
+        )
+
+    def _split_into_lines(self, content: str) -> List[str]:
+        """Split content into lines, preserving line endings.
+        
+        Args:
+            content: Content to split
+            
+        Returns:
+            List of lines
+        """
+        return content.splitlines(keepends=True)
+
+    def _get_line_range(self, lines: List[str], start_line: int, end_line: int) -> str:
+        """Extract a range of lines from content.
+        
+        Args:
+            lines: List of lines
+            start_line: Starting line number (1-based)
+            end_line: Ending line number (1-based)
+            
+        Returns:
+            Content for the specified line range
+        """
+        # Convert to 0-based indexing
+        start_idx = max(0, start_line - 1)
+        end_idx = min(len(lines), end_line)
+        
+        return "".join(lines[start_idx:end_idx])
+
+
+class FallbackParser(BaseParser):
+    """Fallback parser for unsupported languages using simple text chunking."""
+
+    def __init__(self, language: str = "text") -> None:
+        """Initialize fallback parser."""
+        super().__init__(language)
+
+    async def parse_file(self, file_path: Path) -> List[CodeChunk]:
+        """Parse file using simple text chunking."""
+        try:
+            with open(file_path, "r", encoding="utf-8") as f:
+                content = f.read()
+            return await self.parse_content(content, file_path)
+        except Exception:
+            # Return empty list if file can't be read
+            return []
+
+    async def parse_content(self, content: str, file_path: Path) -> List[CodeChunk]:
+        """Parse content using simple text chunking."""
+        if not content.strip():
+            return []
+
+        lines = self._split_into_lines(content)
+        chunks = []
+        
+        # Simple chunking: split into chunks of ~50 lines
+        chunk_size = 50
+        for i in range(0, len(lines), chunk_size):
+            start_line = i + 1
+            end_line = min(i + chunk_size, len(lines))
+            
+            chunk_content = self._get_line_range(lines, start_line, end_line)
+            
+            if chunk_content.strip():
+                chunk = self._create_chunk(
+                    content=chunk_content,
+                    file_path=file_path,
+                    start_line=start_line,
+                    end_line=end_line,
+                    chunk_type="text",
+                )
+                chunks.append(chunk)
+
+        return chunks
+
+    def get_supported_extensions(self) -> List[str]:
+        """Fallback parser supports all extensions."""
+        return ["*"]  # Special marker for "all extensions"