mcp-vector-search 0.9.3__py3-none-any.whl → 0.12.1__py3-none-any.whl

mcp_vector_search/core/directory_index.py

@@ -0,0 +1,318 @@
+"""Directory index for tracking project structure and file relationships."""
+
+import json
+from collections import defaultdict
+from pathlib import Path
+from typing import Any
+
+from loguru import logger
+
+from .models import Directory
+
+
+class DirectoryIndex:
+    """Manages directory structure and file-directory relationships."""
+
+    def __init__(self, index_path: Path) -> None:
+        """Initialize directory index.
+
+        Args:
+            index_path: Path to directory index file (JSON)
+        """
+        self.index_path = index_path
+        self.directories: dict[str, Directory] = {}  # path -> Directory
+        self.file_to_directory: dict[str, str] = {}  # file_path -> directory_path
+        self.directory_files: dict[str, list[str]] = defaultdict(
+            list
+        )  # dir_path -> [file_paths]
+
+    def load(self) -> None:
+        """Load directory index from disk."""
+        if not self.index_path.exists():
+            logger.debug("No directory index found, starting fresh")
+            return
+
+        try:
+            with open(self.index_path) as f:
+                data = json.load(f)
+
+            # Load directories
+            for dir_data in data.get("directories", []):
+                directory = Directory.from_dict(dir_data)
+                self.directories[str(directory.path)] = directory
+
+            # Load file mappings
+            self.file_to_directory = data.get("file_to_directory", {})
+
+            # Rebuild directory_files from file_to_directory
+            self.directory_files = defaultdict(list)
+            for file_path, dir_path in self.file_to_directory.items():
+                self.directory_files[dir_path].append(file_path)
+
+            logger.info(f"Loaded {len(self.directories)} directories from index")
+
+        except Exception as e:
+            logger.error(f"Failed to load directory index: {e}")
+            self.directories = {}
+            self.file_to_directory = {}
+            self.directory_files = defaultdict(list)
+
+    def save(self) -> None:
+        """Save directory index to disk."""
+        try:
+            # Ensure parent directory exists
+            self.index_path.parent.mkdir(parents=True, exist_ok=True)
+
+            data = {
+                "directories": [d.to_dict() for d in self.directories.values()],
+                "file_to_directory": self.file_to_directory,
+            }
+
+            with open(self.index_path, "w") as f:
+                json.dump(data, f, indent=2)
+
+            logger.debug(f"Saved {len(self.directories)} directories to index")
+
+        except Exception as e:
+            logger.error(f"Failed to save directory index: {e}")
+            raise
+
+    def add_directory(self, directory: Directory) -> None:
+        """Add or update a directory in the index.
+
+        Args:
+            directory: Directory to add
+        """
+        dir_path = str(directory.path)
+        self.directories[dir_path] = directory
+
+    def add_file(self, file_path: Path, directory_path: Path) -> None:
+        """Associate a file with its directory.
+
+        Args:
+            file_path: Path to the file
+            directory_path: Path to the directory containing the file
+        """
+        file_path_str = str(file_path)
+        dir_path_str = str(directory_path)
+
+        self.file_to_directory[file_path_str] = dir_path_str
+        if file_path_str not in self.directory_files[dir_path_str]:
+            self.directory_files[dir_path_str].append(file_path_str)
+
+        # Update directory file count
+        if dir_path_str in self.directories:
+            self.directories[dir_path_str].file_count = len(
+                self.directory_files[dir_path_str]
+            )
+
+    def get_directory(self, directory_path: Path) -> Directory | None:
+        """Get directory by path.
+
+        Args:
+            directory_path: Path to directory
+
+        Returns:
+            Directory object or None if not found
+        """
+        return self.directories.get(str(directory_path))
+
+    def get_files_in_directory(self, directory_path: Path) -> list[str]:
+        """Get all files in a directory.
+
+        Args:
+            directory_path: Path to directory
+
+        Returns:
+            List of file paths in the directory
+        """
+        return self.directory_files.get(str(directory_path), [])
+
+    def get_subdirectories(self, directory_path: Path) -> list[Directory]:
+        """Get all immediate subdirectories.
+
+        Args:
+            directory_path: Path to parent directory
+
+        Returns:
+            List of subdirectory objects
+        """
+        parent_path_str = str(directory_path)
+        subdirs = []
+
+        for _dir_path_str, directory in self.directories.items():
+            if directory.parent_path and str(directory.parent_path) == parent_path_str:
+                subdirs.append(directory)
+
+        return subdirs
+
+    def get_root_directories(self) -> list[Directory]:
+        """Get all root-level directories (no parent).
+
+        Returns:
+            List of root directory objects
+        """
+        return [d for d in self.directories.values() if d.parent_path is None]
+
+    def delete_directory(self, directory_path: Path) -> None:
+        """Remove directory and its file associations.
+
+        Args:
+            directory_path: Path to directory to remove
+        """
+        dir_path_str = str(directory_path)
+
+        # Remove directory
+        if dir_path_str in self.directories:
+            del self.directories[dir_path_str]
+
+        # Remove file associations
+        if dir_path_str in self.directory_files:
+            for file_path in self.directory_files[dir_path_str]:
+                if file_path in self.file_to_directory:
+                    del self.file_to_directory[file_path]
+            del self.directory_files[dir_path_str]
+
+    def delete_file(self, file_path: Path) -> None:
+        """Remove file from directory associations.
+
+        Args:
+            file_path: Path to file to remove
+        """
+        file_path_str = str(file_path)
+
+        if file_path_str in self.file_to_directory:
+            dir_path = self.file_to_directory[file_path_str]
+            del self.file_to_directory[file_path_str]
+
+            # Remove from directory_files
+            if dir_path in self.directory_files:
+                self.directory_files[dir_path] = [
+                    f for f in self.directory_files[dir_path] if f != file_path_str
+                ]
+
+                # Update directory file count
+                if dir_path in self.directories:
+                    self.directories[dir_path].file_count = len(
+                        self.directory_files[dir_path]
+                    )
+
+    def rebuild_from_files(
+        self,
+        file_paths: list[Path],
+        root_path: Path,
+        chunk_stats: dict[str, dict] | None = None,
+    ) -> None:
+        """Rebuild directory index from list of files with statistics from chunks.
+
+        Args:
+            file_paths: List of file paths to index
+            root_path: Project root path
+            chunk_stats: Optional dict mapping file_path -> {'chunks': count, 'language': str}
+        """
+        self.directories = {}
+        self.file_to_directory = {}
+        self.directory_files = defaultdict(list)
+
+        # Track all unique directories and their statistics
+        dir_set = set()
+        dir_chunks = defaultdict(int)  # directory -> total chunks
+        dir_languages = defaultdict(
+            lambda: defaultdict(int)
+        )  # directory -> {language: count}
+        dir_modified = defaultdict(float)  # directory -> most recent modification time
+
+        for file_path in file_paths:
+            # Get relative path from root
+            try:
+                rel_path = file_path.relative_to(root_path)
+                parent_dir = rel_path.parent
+
+                # Add all parent directories up to root
+                current = parent_dir
+                while current != Path("."):
+                    dir_set.add(current)
+
+                    # Accumulate statistics up the directory tree
+                    if chunk_stats and str(file_path) in chunk_stats:
+                        stats = chunk_stats[str(file_path)]
+                        dir_chunks[current] += stats.get("chunks", 0)
+                        if "language" in stats:
+                            dir_languages[current][stats["language"]] += stats.get(
+                                "chunks", 0
+                            )
+                        # Track most recent modification time
+                        if "modified" in stats:
+                            dir_modified[current] = max(
+                                dir_modified.get(current, 0), stats["modified"]
+                            )
+
+                    current = current.parent
+
+                # Associate file with its direct parent
+                if parent_dir != Path("."):
+                    self.add_file(rel_path, parent_dir)
+
+            except ValueError:
+                # File not relative to root, skip
+                logger.warning(f"File {file_path} not under root {root_path}")
+                continue
+
+        # Create Directory objects for all directories
+        for dir_path in sorted(dir_set):
+            # Determine parent
+            parent_path = dir_path.parent if dir_path.parent != Path(".") else None
+
+            # Check if it's a package
+            is_package = False
+            full_dir_path = root_path / dir_path
+            if (full_dir_path / "__init__.py").exists():
+                is_package = True
+            elif (full_dir_path / "package.json").exists():
+                is_package = True
+
+            directory = Directory(
+                path=dir_path,
+                name=dir_path.name,
+                parent_path=parent_path,
+                depth=len(dir_path.parts),
+                is_package=is_package,
+                total_chunks=dir_chunks.get(dir_path, 0),
+                languages=dict(dir_languages.get(dir_path, {})),
+                last_modified=dir_modified.get(dir_path),
+            )
+
+            self.add_directory(directory)
+
+        # Update subdirectory counts
+        for directory in self.directories.values():
+            subdirs = self.get_subdirectories(directory.path)
+            directory.subdirectory_count = len(subdirs)
+
+        logger.info(
+            f"Rebuilt directory index with {len(self.directories)} directories, {sum(dir_chunks.values())} total chunks"
+        )
+
+    def get_stats(self) -> dict[str, Any]:
+        """Get directory index statistics.
+
+        Returns:
+            Dictionary with statistics
+        """
+        return {
+            "total_directories": len(self.directories),
+            "total_files": len(self.file_to_directory),
+            "root_directories": len(self.get_root_directories()),
+            "packages": sum(1 for d in self.directories.values() if d.is_package),
+        }
+
+    def reset(self) -> None:
+        """Clear all directory data."""
+        self.directories = {}
+        self.file_to_directory = {}
+        self.directory_files = defaultdict(list)
+
+        if self.index_path.exists():
+            self.index_path.unlink()
+
+        logger.info("Directory index reset")

mcp_vector_search/core/indexer.py

@@ -15,8 +15,9 @@ from ..parsers.registry import get_parser_registry
 from ..utils.gitignore import create_gitignore_parser
 from ..utils.monorepo import MonorepoDetector
 from .database import VectorDatabase
+from .directory_index import DirectoryIndex
 from .exceptions import ParsingError
-from .models import CodeChunk
+from .models import CodeChunk, IndexStats
 
 
 class SemanticIndexer:
@@ -29,6 +30,7 @@ class SemanticIndexer:
         file_extensions: list[str],
         max_workers: int | None = None,
         batch_size: int = 10,
+        debug: bool = False,
     ) -> None:
         """Initialize semantic indexer.
 
@@ -38,12 +40,14 @@ class SemanticIndexer:
             file_extensions: File extensions to index
             max_workers: Maximum number of worker threads for parallel processing
             batch_size: Number of files to process in each batch
+            debug: Enable debug output for hierarchy building
         """
         self.database = database
         self.project_root = project_root
         self.file_extensions = {ext.lower() for ext in file_extensions}
         self.parser_registry = get_parser_registry()
         self._ignore_patterns = set(DEFAULT_IGNORE_PATTERNS)
+        self.debug = debug
 
         # Safely get event loop for max_workers
         try:
@@ -81,6 +85,13 @@ class SemanticIndexer:
             for sp in subprojects:
                 logger.debug(f"  - {sp.name} ({sp.relative_path})")
 
+        # Initialize directory index
+        self.directory_index = DirectoryIndex(
+            project_root / ".mcp-vector-search" / "directory_index.json"
+        )
+        # Load existing directory index
+        self.directory_index.load()
+
     async def index_project(
         self,
         force_reindex: bool = False,
@@ -156,6 +167,39 @@ class SemanticIndexer:
 
             self._save_index_metadata(metadata)
 
+            # Rebuild directory index from successfully indexed files
+            try:
+                logger.debug("Rebuilding directory index...")
+                # We don't have chunk counts here, but we have file modification times
+                # Build a simple stats dict with file mod times for recency tracking
+                chunk_stats = {}
+                for file_path in files_to_index:
+                    try:
+                        mtime = os.path.getmtime(file_path)
+                        # For now, just track modification time
+                        # Chunk counts will be aggregated from the database later if needed
+                        chunk_stats[str(file_path)] = {
+                            "modified": mtime,
+                            "chunks": 1,  # Placeholder - real count from chunks
+                        }
+                    except OSError:
+                        pass
+
+                self.directory_index.rebuild_from_files(
+                    files_to_index, self.project_root, chunk_stats=chunk_stats
+                )
+                self.directory_index.save()
+                dir_stats = self.directory_index.get_stats()
+                logger.info(
+                    f"Directory index updated: {dir_stats['total_directories']} directories, "
+                    f"{dir_stats['total_files']} files"
+                )
+            except Exception as e:
+                logger.error(f"Failed to update directory index: {e}")
+                import traceback
+
+                logger.debug(traceback.format_exc())
+
         logger.info(
             f"Indexing complete: {indexed_count} files indexed, {failed_count} failed"
         )
@@ -307,6 +351,16 @@ class SemanticIndexer:
             # Build hierarchical relationships between chunks
             chunks_with_hierarchy = self._build_chunk_hierarchy(chunks)
 
+            # Debug: Check if hierarchy was built
+            methods_with_parents = sum(
+                1
+                for c in chunks_with_hierarchy
+                if c.chunk_type in ("method", "function") and c.parent_chunk_id
+            )
+            logger.debug(
+                f"After hierarchy build: {methods_with_parents}/{len([c for c in chunks_with_hierarchy if c.chunk_type in ('method', 'function')])} methods have parents"
+            )
+
             # Add chunks to database
             await self.database.add_chunks(chunks_with_hierarchy)
 
@@ -396,7 +450,11 @@ class SemanticIndexer:
             # Filter out ignored directories IN-PLACE to prevent os.walk from traversing them
             # This is much more efficient than checking every file in ignored directories
             # PERFORMANCE: Pass is_directory=True hint to skip filesystem stat() calls
-            dirs[:] = [d for d in dirs if not self._should_ignore_path(root_path / d, is_directory=True)]
+            dirs[:] = [
+                d
+                for d in dirs
+                if not self._should_ignore_path(root_path / d, is_directory=True)
+            ]
 
             # Check each file in the current directory
             # PERFORMANCE: skip_file_check=True because os.walk guarantees these are files
@@ -442,7 +500,9 @@ class SemanticIndexer:
 
         return self._indexable_files_cache
 
-    def _should_index_file(self, file_path: Path, skip_file_check: bool = False) -> bool:
+    def _should_index_file(
+        self, file_path: Path, skip_file_check: bool = False
+    ) -> bool:
         """Check if a file should be indexed.
 
         Args:
@@ -478,7 +538,9 @@ class SemanticIndexer:
 
         return True
 
-    def _should_ignore_path(self, file_path: Path, is_directory: bool | None = None) -> bool:
+    def _should_ignore_path(
+        self, file_path: Path, is_directory: bool | None = None
+    ) -> bool:
         """Check if a path should be ignored.
 
         Args:
@@ -491,7 +553,9 @@ class SemanticIndexer:
         try:
             # First check gitignore rules if available
             # PERFORMANCE: Pass is_directory hint to avoid redundant stat() calls
-            if self.gitignore_parser and self.gitignore_parser.is_ignored(file_path, is_directory=is_directory):
+            if self.gitignore_parser and self.gitignore_parser.is_ignored(
+                file_path, is_directory=is_directory
+            ):
                 logger.debug(f"Path ignored by .gitignore: {file_path}")
                 return True
 
@@ -630,24 +694,34 @@ class SemanticIndexer:
             # If we can't parse versions, be safe and reindex
             return True
 
-    async def get_indexing_stats(self) -> dict:
+    async def get_indexing_stats(self, db_stats: IndexStats | None = None) -> dict:
         """Get statistics about the indexing process.
 
+        Args:
+            db_stats: Optional pre-fetched database stats to avoid duplicate queries
+
         Returns:
             Dictionary with indexing statistics
+
+        Note:
+            Uses database statistics only for performance on large projects.
+            Filesystem scanning would timeout on 100K+ file projects.
+            Pass db_stats parameter to avoid calling database.get_stats() twice.
         """
         try:
-            # Get database stats
-            db_stats = await self.database.get_stats()
-
-            # Count indexable files asynchronously without blocking
-            indexable_files = await self._find_indexable_files_async()
+            # Get database stats if not provided (fast, no filesystem scan)
+            if db_stats is None:
+                db_stats = await self.database.get_stats()
 
+            # Use database stats for all file counts
+            # This avoids expensive filesystem scans on large projects
             return {
-                "total_indexable_files": len(indexable_files),
+                "total_indexable_files": db_stats.total_files,
                 "indexed_files": db_stats.total_files,
+                "total_files": db_stats.total_files,  # For backward compatibility
                 "total_chunks": db_stats.total_chunks,
                 "languages": db_stats.languages,
+                "file_types": db_stats.file_types,  # Include file type distribution
                 "file_extensions": list(self.file_extensions),
                 "ignore_patterns": list(self._ignore_patterns),
                 "parser_info": self.parser_registry.get_parser_info(),
@@ -659,6 +733,7 @@ class SemanticIndexer:
                 "error": str(e),
                 "total_indexable_files": 0,
                 "indexed_files": 0,
+                "total_files": 0,
                 "total_chunks": 0,
             }
 
@@ -752,9 +827,14 @@ class SemanticIndexer:
 
                     # Save error to error log file
                     try:
-                        error_log_path = self.project_root / ".mcp-vector-search" / "indexing_errors.log"
+                        error_log_path = (
+                            self.project_root
+                            / ".mcp-vector-search"
+                            / "indexing_errors.log"
+                        )
                         with open(error_log_path, "a", encoding="utf-8") as f:
                             from datetime import datetime
+
                             timestamp = datetime.now().isoformat()
                             f.write(f"[{timestamp}] {error_msg}\n")
                     except Exception as log_err:
@@ -787,22 +867,54 @@ class SemanticIndexer:
 
         # Group chunks by type and name
         module_chunks = [c for c in chunks if c.chunk_type in ("module", "imports")]
-        class_chunks = [c for c in chunks if c.chunk_type in ("class", "interface", "mixin")]
-        function_chunks = [c for c in chunks if c.chunk_type in ("function", "method", "constructor")]
+        class_chunks = [
+            c for c in chunks if c.chunk_type in ("class", "interface", "mixin")
+        ]
+        function_chunks = [
+            c for c in chunks if c.chunk_type in ("function", "method", "constructor")
+        ]
+
+        # DEBUG: Print what we have (if debug enabled)
+        if self.debug:
+            import sys
+
+            print(
+                f"\n[DEBUG] Building hierarchy: {len(module_chunks)} modules, {len(class_chunks)} classes, {len(function_chunks)} functions",
+                file=sys.stderr,
+            )
+            if class_chunks:
+                print(
+                    f"[DEBUG] Class names: {[c.class_name for c in class_chunks[:5]]}",
+                    file=sys.stderr,
+                )
+            if function_chunks:
+                print(
+                    f"[DEBUG] First 5 functions with class_name: {[(f.function_name, f.class_name) for f in function_chunks[:5]]}",
+                    file=sys.stderr,
+                )
 
         # Build relationships
         for func in function_chunks:
             if func.class_name:
                 # Find parent class
                 parent_class = next(
-                    (c for c in class_chunks if c.class_name == func.class_name),
-                    None
+                    (c for c in class_chunks if c.class_name == func.class_name), None
                 )
                 if parent_class:
                     func.parent_chunk_id = parent_class.chunk_id
                     func.chunk_depth = parent_class.chunk_depth + 1
                     if func.chunk_id not in parent_class.child_chunk_ids:
                         parent_class.child_chunk_ids.append(func.chunk_id)
+                    if self.debug:
+                        import sys
+
+                        print(
+                            f"[DEBUG] ✓ Linked '{func.function_name}' to class '{parent_class.class_name}'",
+                            file=sys.stderr,
+                        )
+                    logger.debug(
+                        f"Linked method '{func.function_name}' (ID: {func.chunk_id[:8]}) to class '{parent_class.class_name}' (ID: {parent_class.chunk_id[:8]})"
+                    )
             else:
                 # Top-level function
                 if not func.chunk_depth:
@@ -828,19 +940,34 @@ class SemanticIndexer:
             if not mod.chunk_depth:
                 mod.chunk_depth = 0
 
+        # DEBUG: Print summary
+        if self.debug:
+            import sys
+
+            funcs_with_parents = sum(1 for f in function_chunks if f.parent_chunk_id)
+            classes_with_parents = sum(1 for c in class_chunks if c.parent_chunk_id)
+            print(
+                f"[DEBUG] Hierarchy built: {funcs_with_parents}/{len(function_chunks)} functions linked, {classes_with_parents}/{len(class_chunks)} classes linked\n",
+                file=sys.stderr,
+            )
+
         return chunks
 
     def _write_indexing_run_header(self) -> None:
         """Write version and timestamp header to error log at start of indexing run."""
         try:
-            error_log_path = self.project_root / ".mcp-vector-search" / "indexing_errors.log"
+            error_log_path = (
+                self.project_root / ".mcp-vector-search" / "indexing_errors.log"
+            )
             error_log_path.parent.mkdir(parents=True, exist_ok=True)
 
             with open(error_log_path, "a", encoding="utf-8") as f:
                 timestamp = datetime.now(UTC).isoformat()
                 separator = "=" * 80
                 f.write(f"\n{separator}\n")
-                f.write(f"[{timestamp}] Indexing run started - mcp-vector-search v{__version__}\n")
+                f.write(
+                    f"[{timestamp}] Indexing run started - mcp-vector-search v{__version__}\n"
+                )
                 f.write(f"{separator}\n")
         except Exception as e:
             logger.debug(f"Failed to write indexing run header: {e}")

mcp_vector_search/core/models.py

@@ -206,6 +206,67 @@ class IndexStats(BaseModel):
         }
 
 
+@dataclass
+class Directory:
+    """Represents a directory in the project structure."""
+
+    path: Path  # Relative path from project root
+    name: str  # Directory name
+    parent_path: Path | None = None  # Parent directory path (None for root)
+    file_count: int = 0  # Number of files directly in this directory
+    subdirectory_count: int = 0  # Number of subdirectories
+    total_chunks: int = 0  # Total code chunks in this directory (recursive)
+    languages: dict[str, int] = None  # Language distribution in this directory
+    depth: int = 0  # Depth from project root (0 = root)
+    is_package: bool = False  # True if contains __init__.py or package.json
+    last_modified: float | None = (
+        None  # Most recent file modification time (unix timestamp)
+    )
+
+    def __post_init__(self) -> None:
+        """Initialize default values and generate directory ID."""
+        if self.languages is None:
+            self.languages = {}
+
+    @property
+    def id(self) -> str:
+        """Generate unique ID for this directory."""
+        import hashlib
+
+        return hashlib.sha256(str(self.path).encode()).hexdigest()[:16]
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary for storage."""
+        return {
+            "path": str(self.path),
+            "name": self.name,
+            "parent_path": str(self.parent_path) if self.parent_path else None,
+            "file_count": self.file_count,
+            "subdirectory_count": self.subdirectory_count,
+            "total_chunks": self.total_chunks,
+            "languages": self.languages,
+            "depth": self.depth,
+            "is_package": self.is_package,
+            "last_modified": self.last_modified,
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "Directory":
+        """Create from dictionary."""
+        return cls(
+            path=Path(data["path"]),
+            name=data["name"],
+            parent_path=Path(data["parent_path"]) if data.get("parent_path") else None,
+            file_count=data.get("file_count", 0),
+            subdirectory_count=data.get("subdirectory_count", 0),
+            total_chunks=data.get("total_chunks", 0),
+            languages=data.get("languages", {}),
+            depth=data.get("depth", 0),
+            is_package=data.get("is_package", False),
+            last_modified=data.get("last_modified"),
+        )
+
+
 class ProjectInfo(BaseModel):
     """Information about a project."""