mcp-code-indexer 4.0.2__py3-none-any.whl → 4.2.0__py3-none-any.whl

mcp_code_indexer/vector_mode/daemon.py

@@ -0,0 +1,335 @@
+"""
+Vector Mode Daemon.
+
+Runs as a background process to monitor file changes and maintain vector indexes.
+Handles embedding generation, change detection, and vector database synchronization.
+"""
+
+import asyncio
+import logging
+import signal
+import sys
+from pathlib import Path
+from typing import Optional, Set
+import json
+import time
+
+from ..database.database import DatabaseManager
+from .config import VectorConfig, load_vector_config
+
+logger = logging.getLogger(__name__)
+
+class VectorDaemon:
+    """
+    Background daemon for vector mode operations.
+    
+    Monitors file changes, generates embeddings, and maintains vector indexes
+    for all projects with vector mode enabled.
+    """
+    
+    def __init__(
+        self,
+        config: VectorConfig,
+        db_manager: DatabaseManager,
+        cache_dir: Path,
+    ):
+        """Initialize vector daemon."""
+        self.config = config
+        self.db_manager = db_manager
+        self.cache_dir = cache_dir
+        self.is_running = False
+        self.shutdown_requested = False
+        
+        # Process tracking
+        self.monitored_projects: Set[str] = set()
+        self.processing_queue: asyncio.Queue = asyncio.Queue(maxsize=config.max_queue_size)
+        self.workers: list[asyncio.Task] = []
+        
+        # Statistics
+        self.stats = {
+            "start_time": time.time(),
+            "files_processed": 0,
+            "embeddings_generated": 0,
+            "errors_count": 0,
+            "last_activity": time.time(),
+        }
+        
+        # Setup signal handlers
+        self._setup_signal_handlers()
+        
+    def _setup_signal_handlers(self) -> None:
+        """Setup signal handlers for graceful shutdown."""
+        try:
+            signal.signal(signal.SIGTERM, self._signal_handler)
+            signal.signal(signal.SIGINT, self._signal_handler)
+            if hasattr(signal, 'SIGHUP'):
+                signal.signal(signal.SIGHUP, self._signal_handler)
+        except Exception as e:
+            logger.warning(f"Could not setup signal handlers: {e}")
+    
+    def _signal_handler(self, signum: int, frame) -> None:
+        """Handle shutdown signals."""
+        logger.info(f"Received signal {signum}, initiating graceful shutdown")
+        self.shutdown_requested = True
+    
+    async def start(self) -> None:
+        """Start the vector daemon."""
+        if self.is_running:
+            logger.warning("Daemon is already running")
+            return
+        
+        self.is_running = True
+        logger.info(
+            "Starting vector daemon",
+            extra={
+                "structured_data": {
+                    "config": {
+                        "worker_count": self.config.worker_count,
+                        "batch_size": self.config.batch_size,
+                        "poll_interval": self.config.daemon_poll_interval,
+                    }
+                }
+            }
+        )
+        
+        try:
+            # Start worker tasks
+            for i in range(self.config.worker_count):
+                worker = asyncio.create_task(self._worker(f"worker-{i}"))
+                self.workers.append(worker)
+            
+            # Start monitoring tasks
+            monitor_task = asyncio.create_task(self._monitor_projects())
+            stats_task = asyncio.create_task(self._stats_reporter())
+            
+            # Wait for shutdown signal
+            await self._run_until_shutdown()
+            
+        except Exception as e:
+            logger.error(f"Daemon error: {e}", exc_info=True)
+            self.stats["errors_count"] += 1
+        finally:
+            await self._cleanup()
+    
+    async def _run_until_shutdown(self) -> None:
+        """Run daemon until shutdown is requested."""
+        while not self.shutdown_requested:
+            try:
+                await asyncio.sleep(1.0)
+            except asyncio.CancelledError:
+                break
+    
+    async def _monitor_projects(self) -> None:
+        """Monitor projects for vector indexing requirements."""
+        logger.info("Starting project monitoring")
+        
+        while not self.shutdown_requested:
+            try:
+                # Get all projects that need vector indexing
+                projects = await self.db_manager.get_all_projects()
+                
+                for project in projects:
+                    if project.name not in self.monitored_projects:
+                        logger.info(f"Adding project to monitoring: {project.name}")
+                        self.monitored_projects.add(project.name)
+                        
+                        # Queue initial indexing task
+                        await self._queue_project_scan(project.name, project.folder_path)
+                
+                await asyncio.sleep(self.config.daemon_poll_interval)
+                
+            except Exception as e:
+                logger.error(f"Error in project monitoring: {e}")
+                self.stats["errors_count"] += 1
+                await asyncio.sleep(5.0)  # Back off on error
+    
+    async def _queue_project_scan(self, project_name: str, folder_path: str) -> None:
+        """Queue a project for scanning and indexing."""
+        task = {
+            "type": "scan_project",
+            "project_name": project_name,
+            "folder_path": folder_path,
+            "timestamp": time.time(),
+        }
+        
+        try:
+            await self.processing_queue.put(task)
+            logger.debug(f"Queued project scan: {project_name}")
+        except asyncio.QueueFull:
+            logger.warning(f"Processing queue full, dropping scan task for {project_name}")
+    
+    async def _worker(self, worker_id: str) -> None:
+        """Worker task to process queued items."""
+        logger.info(f"Starting worker: {worker_id}")
+        
+        while not self.shutdown_requested:
+            try:
+                # Get task from queue with timeout
+                try:
+                    task = await asyncio.wait_for(
+                        self.processing_queue.get(), 
+                        timeout=5.0
+                    )
+                except asyncio.TimeoutError:
+                    continue
+                
+                # Process the task
+                await self._process_task(task, worker_id)
+                self.stats["last_activity"] = time.time()
+                
+            except Exception as e:
+                logger.error(f"Worker {worker_id} error: {e}")
+                self.stats["errors_count"] += 1
+                await asyncio.sleep(1.0)  # Brief pause on error
+    
+    async def _process_task(self, task: dict, worker_id: str) -> None:
+        """Process a queued task."""
+        task_type = task.get("type")
+        
+        if task_type == "scan_project":
+            await self._process_project_scan(task, worker_id)
+        else:
+            logger.warning(f"Unknown task type: {task_type}")
+    
+    async def _process_project_scan(self, task: dict, worker_id: str) -> None:
+        """Process a project scan task."""
+        project_name = task["project_name"]
+        folder_path = task["folder_path"]
+        
+        logger.debug(f"Worker {worker_id} processing project: {project_name}")
+        
+        try:
+            # Check if vector mode components are available
+            # For now, just log that we would process this project
+            logger.info(
+                f"Vector processing for project {project_name}",
+                extra={
+                    "structured_data": {
+                        "project_name": project_name,
+                        "folder_path": folder_path,
+                        "worker_id": worker_id,
+                    }
+                }
+            )
+            
+            self.stats["files_processed"] += 1
+            
+            # TODO: Implement actual vector processing:
+            # 1. Scan for file changes using Merkle tree
+            # 2. Chunk modified files using AST
+            # 3. Apply secret redaction
+            # 4. Generate embeddings via Voyage
+            # 5. Store in Turbopuffer
+            # 6. Update database metadata
+            
+        except Exception as e:
+            logger.error(f"Error processing project {project_name}: {e}")
+            self.stats["errors_count"] += 1
+    
+    async def _stats_reporter(self) -> None:
+        """Periodically report daemon statistics."""
+        while not self.shutdown_requested:
+            try:
+                uptime = time.time() - self.stats["start_time"]
+                
+                logger.info(
+                    "Daemon statistics",
+                    extra={
+                        "structured_data": {
+                            "uptime_seconds": uptime,
+                            "monitored_projects": len(self.monitored_projects),
+                            "queue_size": self.processing_queue.qsize(),
+                            "files_processed": self.stats["files_processed"],
+                            "embeddings_generated": self.stats["embeddings_generated"],
+                            "errors_count": self.stats["errors_count"],
+                        }
+                    }
+                )
+                
+                await asyncio.sleep(60.0)  # Report every minute
+                
+            except Exception as e:
+                logger.error(f"Error in stats reporting: {e}")
+                await asyncio.sleep(10.0)
+    
+    async def _cleanup(self) -> None:
+        """Clean up resources and shut down workers."""
+        logger.info("Starting daemon cleanup")
+        self.is_running = False
+        
+        # Cancel all workers
+        for worker in self.workers:
+            worker.cancel()
+        
+        # Wait for workers to finish
+        if self.workers:
+            await asyncio.gather(*self.workers, return_exceptions=True)
+        
+        logger.info("Vector daemon shutdown complete")
+    
+    def get_status(self) -> dict:
+        """Get current daemon status."""
+        return {
+            "is_running": self.is_running,
+            "uptime": time.time() - self.stats["start_time"] if self.is_running else 0,
+            "monitored_projects": len(self.monitored_projects),
+            "queue_size": self.processing_queue.qsize(),
+            "stats": self.stats.copy(),
+        }
+
+async def start_vector_daemon(
+    config_path: Optional[Path] = None,
+    db_path: Optional[Path] = None,
+    cache_dir: Optional[Path] = None,
+) -> None:
+    """Start the vector daemon process."""
+    
+    # Load configuration
+    config = load_vector_config(config_path)
+    
+    # Setup database
+    if db_path is None:
+        db_path = Path.home() / ".mcp-code-index" / "tracker.db"
+    if cache_dir is None:
+        cache_dir = Path.home() / ".mcp-code-index" / "cache"
+    
+    db_manager = DatabaseManager(db_path)
+    await db_manager.initialize()
+    
+    # Create and start daemon
+    daemon = VectorDaemon(config, db_manager, cache_dir)
+    
+    try:
+        await daemon.start()
+    finally:
+        # Clean up database connections
+        await db_manager.close_pool()
+
+def main() -> None:
+    """CLI entry point for vector daemon."""
+    import argparse
+    
+    parser = argparse.ArgumentParser(description="MCP Code Indexer Vector Daemon")
+    parser.add_argument("--config", type=Path, help="Path to config file")
+    parser.add_argument("--db-path", type=Path, help="Path to database")
+    parser.add_argument("--cache-dir", type=Path, help="Cache directory")
+    parser.add_argument("--log-level", default="INFO", help="Logging level")
+    
+    args = parser.parse_args()
+    
+    # Setup logging
+    logging.basicConfig(
+        level=getattr(logging, args.log_level.upper()),
+        format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+    )
+    
+    try:
+        asyncio.run(start_vector_daemon(args.config, args.db_path, args.cache_dir))
+    except KeyboardInterrupt:
+        logger.info("Daemon interrupted by user")
+    except Exception as e:
+        logger.error(f"Daemon failed: {e}", exc_info=True)
+        sys.exit(1)
+
+if __name__ == "__main__":
+    main()

mcp_code_indexer/vector_mode/monitoring/__init__.py

@@ -0,0 +1,19 @@
+"""
+File system monitoring for vector mode.
+
+Provides real-time file change detection using watchdog and efficient
+change tracking using Merkle trees.
+"""
+
+from .file_watcher import FileWatcher
+from .merkle_tree import MerkleTree, MerkleNode
+from .change_detector import ChangeDetector, FileChange, ChangeType
+
+__all__ = [
+    "FileWatcher",
+    "MerkleTree", 
+    "MerkleNode",
+    "ChangeDetector",
+    "FileChange",
+    "ChangeType",
+]

mcp_code_indexer/vector_mode/monitoring/change_detector.py

@@ -0,0 +1,312 @@
+"""
+Change detection utilities for file system monitoring.
+
+Provides high-level change detection and classification for the vector mode
+file monitoring system.
+"""
+
+import logging
+from enum import Enum
+from typing import List, Dict, Set, Optional, NamedTuple
+from pathlib import Path
+from dataclasses import dataclass
+from datetime import datetime
+
+logger = logging.getLogger(__name__)
+
+class ChangeType(str, Enum):
+    """Types of file system changes."""
+    CREATED = "created"
+    MODIFIED = "modified"
+    DELETED = "deleted"
+    MOVED = "moved"
+
+class FileChange(NamedTuple):
+    """Represents a file system change."""
+    path: str
+    change_type: ChangeType
+    timestamp: datetime
+    old_path: Optional[str] = None  # For moves
+    size: Optional[int] = None
+    hash: Optional[str] = None
+
+@dataclass
+class ChangeStats:
+    """Statistics about detected changes."""
+    total_changes: int = 0
+    creates: int = 0
+    modifications: int = 0
+    deletions: int = 0
+    moves: int = 0
+    start_time: Optional[datetime] = None
+    last_change: Optional[datetime] = None
+
+class ChangeDetector:
+    """
+    High-level change detection and classification.
+    
+    Processes raw file system events and provides structured change information
+    for the vector indexing pipeline.
+    """
+    
+    def __init__(
+        self,
+        project_root: Path,
+        ignore_patterns: Optional[List[str]] = None,
+        debounce_interval: float = 0.1,
+    ):
+        """
+        Initialize change detector.
+        
+        Args:
+            project_root: Root directory to monitor
+            ignore_patterns: Patterns to ignore (glob-style)
+            debounce_interval: Minimum time between processing same file
+        """
+        self.project_root = Path(project_root).resolve()
+        self.ignore_patterns = ignore_patterns or [
+            "*.log", "*.tmp", "*~", ".git/*", "__pycache__/*", 
+            "node_modules/*", "*.pyc", "*.pyo", ".DS_Store", "Thumbs.db"
+        ]
+        self.debounce_interval = debounce_interval
+        
+        # Change tracking
+        self.recent_changes: List[FileChange] = []
+        self.pending_changes: Dict[str, FileChange] = {}
+        self.last_change_time: Dict[str, datetime] = {}
+        
+        # Statistics
+        self.stats = ChangeStats(start_time=datetime.utcnow())
+        
+        # Compile ignore patterns for performance
+        import fnmatch
+        self._compiled_patterns = [
+            fnmatch.translate(pattern) for pattern in self.ignore_patterns
+        ]
+    
+    def should_ignore_path(self, path: Path) -> bool:
+        """Check if a path should be ignored based on patterns."""
+        try:
+            relative_path = path.relative_to(self.project_root)
+            path_str = str(relative_path)
+            
+            import re
+            for pattern in self._compiled_patterns:
+                if re.match(pattern, path_str):
+                    return True
+            
+            return False
+            
+        except ValueError:
+            # Path is not relative to project root
+            return True
+    
+    def _should_debounce(self, file_path: str) -> bool:
+        """Check if change should be debounced."""
+        now = datetime.utcnow()
+        
+        if file_path in self.last_change_time:
+            elapsed = (now - self.last_change_time[file_path]).total_seconds()
+            if elapsed < self.debounce_interval:
+                return True
+        
+        self.last_change_time[file_path] = now
+        return False
+    
+    def _get_file_info(self, path: Path) -> Dict[str, Optional[int]]:
+        """Get file information (size, etc.)."""
+        try:
+            if path.exists() and path.is_file():
+                stat = path.stat()
+                return {"size": stat.st_size}
+            else:
+                return {"size": None}
+        except (OSError, PermissionError):
+            return {"size": None}
+    
+    def _classify_change(
+        self,
+        path: Path,
+        event_type: str,
+        old_path: Optional[Path] = None
+    ) -> Optional[FileChange]:
+        """Classify a file system event into a structured change."""
+        
+        # Convert to relative path
+        try:
+            relative_path = str(path.relative_to(self.project_root))
+        except ValueError:
+            # Path outside project root
+            return None
+        
+        # Check if should be ignored
+        if self.should_ignore_path(path):
+            logger.debug(f"Ignoring change to {relative_path} (matches ignore pattern)")
+            return None
+        
+        # Check debouncing
+        if self._should_debounce(relative_path):
+            logger.debug(f"Debouncing change to {relative_path}")
+            return None
+        
+        # Get file info
+        file_info = self._get_file_info(path)
+        
+        # Map event types to change types
+        if event_type in ["created", "added"]:
+            change_type = ChangeType.CREATED
+        elif event_type in ["modified", "changed"]:
+            change_type = ChangeType.MODIFIED
+        elif event_type in ["deleted", "removed"]:
+            change_type = ChangeType.DELETED
+        elif event_type in ["moved", "renamed"]:
+            change_type = ChangeType.MOVED
+        else:
+            logger.warning(f"Unknown event type: {event_type}")
+            return None
+        
+        # Create change object
+        old_relative_path = None
+        if old_path:
+            try:
+                old_relative_path = str(old_path.relative_to(self.project_root))
+            except ValueError:
+                pass
+        
+        change = FileChange(
+            path=relative_path,
+            change_type=change_type,
+            timestamp=datetime.utcnow(),
+            old_path=old_relative_path,
+            size=file_info.get("size"),
+            hash=None  # Will be computed later if needed
+        )
+        
+        return change
+    
+    def process_fs_event(
+        self,
+        event_type: str,
+        path: Path,
+        old_path: Optional[Path] = None
+    ) -> Optional[FileChange]:
+        """
+        Process a file system event and return structured change.
+        
+        Args:
+            event_type: Type of event (created, modified, deleted, moved)
+            path: Path that changed
+            old_path: Old path (for moves)
+            
+        Returns:
+            FileChange object or None if ignored
+        """
+        change = self._classify_change(path, event_type, old_path)
+        
+        if change:
+            self.recent_changes.append(change)
+            
+            # Update statistics
+            self.stats.total_changes += 1
+            self.stats.last_change = change.timestamp
+            
+            if change.change_type == ChangeType.CREATED:
+                self.stats.creates += 1
+            elif change.change_type == ChangeType.MODIFIED:
+                self.stats.modifications += 1
+            elif change.change_type == ChangeType.DELETED:
+                self.stats.deletions += 1
+            elif change.change_type == ChangeType.MOVED:
+                self.stats.moves += 1
+            
+            logger.info(f"Detected change: {change.change_type.value} {change.path}")
+        
+        return change
+    
+    def get_recent_changes(
+        self,
+        limit: Optional[int] = None,
+        change_types: Optional[List[ChangeType]] = None
+    ) -> List[FileChange]:
+        """
+        Get recent changes with optional filtering.
+        
+        Args:
+            limit: Maximum number of changes to return
+            change_types: Filter by change types
+            
+        Returns:
+            List of recent changes
+        """
+        changes = self.recent_changes
+        
+        # Filter by change types
+        if change_types:
+            changes = [c for c in changes if c.change_type in change_types]
+        
+        # Sort by timestamp (most recent first)
+        changes = sorted(changes, key=lambda c: c.timestamp, reverse=True)
+        
+        # Apply limit
+        if limit:
+            changes = changes[:limit]
+        
+        return changes
+    
+    def clear_recent_changes(self) -> int:
+        """Clear recent changes and return count cleared."""
+        count = len(self.recent_changes)
+        self.recent_changes.clear()
+        return count
+    
+    def get_changes_since(self, since: datetime) -> List[FileChange]:
+        """Get all changes since a specific timestamp."""
+        return [
+            change for change in self.recent_changes
+            if change.timestamp >= since
+        ]
+    
+    def get_stats(self) -> ChangeStats:
+        """Get change detection statistics."""
+        return self.stats
+    
+    def reset_stats(self) -> None:
+        """Reset change detection statistics."""
+        self.stats = ChangeStats(start_time=datetime.utcnow())
+    
+    def get_changed_files(self, since: Optional[datetime] = None) -> Set[str]:
+        """Get set of file paths that have changed."""
+        changes = self.recent_changes
+        
+        if since:
+            changes = [c for c in changes if c.timestamp >= since]
+        
+        # Collect unique file paths
+        changed_files = set()
+        for change in changes:
+            changed_files.add(change.path)
+            if change.old_path:  # For moves
+                changed_files.add(change.old_path)
+        
+        return changed_files
+    
+    def is_code_file(self, path: str) -> bool:
+        """Check if a file is likely a code file."""
+        code_extensions = {
+            '.py', '.js', '.ts', '.jsx', '.tsx', '.java', '.cpp', '.c', '.h',
+            '.cs', '.php', '.rb', '.go', '.rs', '.swift', '.kt', '.scala',
+            '.clj', '.cljs', '.hs', '.ml', '.fs', '.ex', '.exs', '.cr',
+            '.dart', '.lua', '.pl', '.sh', '.bash', '.zsh', '.fish',
+            '.sql', '.r', '.m', '.mm', '.vim', '.el', '.lisp', '.scm'
+        }
+        
+        return Path(path).suffix.lower() in code_extensions
+    
+    def get_code_changes(self, since: Optional[datetime] = None) -> List[FileChange]:
+        """Get changes to code files only."""
+        changes = self.get_recent_changes()
+        
+        if since:
+            changes = [c for c in changes if c.timestamp >= since]
+        
+        return [c for c in changes if self.is_code_file(c.path)]