mcp-vector-search 0.9.2__py3-none-any.whl → 0.12.0__py3-none-any.whl

mcp_vector_search/core/database.py

@@ -1,6 +1,7 @@
 """Database abstraction and ChromaDB implementation for MCP Vector Search."""
 
 import asyncio
+import json
 import shutil
 from abc import ABC, abstractmethod
 from pathlib import Path
@@ -273,6 +274,19 @@ class ChromaVectorDatabase(VectorDatabase):
                     "class_name": chunk.class_name or "",
                     "docstring": chunk.docstring or "",
                     "complexity_score": chunk.complexity_score,
+                    # Hierarchy fields (convert lists to JSON strings for ChromaDB)
+                    "chunk_id": chunk.chunk_id or "",
+                    "parent_chunk_id": chunk.parent_chunk_id or "",
+                    "child_chunk_ids": json.dumps(chunk.child_chunk_ids or []),
+                    "chunk_depth": chunk.chunk_depth,
+                    # Additional metadata (convert lists/dicts to JSON strings)
+                    "decorators": json.dumps(chunk.decorators or []),
+                    "parameters": json.dumps(chunk.parameters or []),
+                    "return_type": chunk.return_type or "",
+                    "type_annotations": json.dumps(chunk.type_annotations or {}),
+                    # Monorepo support
+                    "subproject_name": chunk.subproject_name or "",
+                    "subproject_path": chunk.subproject_path or "",
                 }
                 metadatas.append(metadata)
 
@@ -497,6 +511,23 @@ class ChromaVectorDatabase(VectorDatabase):
                     metadata = results["metadatas"][i]
                     content = results["documents"][i]
 
+                    # Parse JSON strings back to lists/dicts
+                    child_chunk_ids = metadata.get("child_chunk_ids", "[]")
+                    if isinstance(child_chunk_ids, str):
+                        child_chunk_ids = json.loads(child_chunk_ids)
+
+                    decorators = metadata.get("decorators", "[]")
+                    if isinstance(decorators, str):
+                        decorators = json.loads(decorators)
+
+                    parameters = metadata.get("parameters", "[]")
+                    if isinstance(parameters, str):
+                        parameters = json.loads(parameters)
+
+                    type_annotations = metadata.get("type_annotations", "{}")
+                    if isinstance(type_annotations, str):
+                        type_annotations = json.loads(type_annotations)
+
                     chunk = CodeChunk(
                         content=content,
                         file_path=Path(metadata["file_path"]),
@@ -511,12 +542,12 @@ class ChromaVectorDatabase(VectorDatabase):
                         complexity_score=metadata.get("complexity_score", 0.0),
                         chunk_id=metadata.get("chunk_id"),
                         parent_chunk_id=metadata.get("parent_chunk_id"),
-                        child_chunk_ids=metadata.get("child_chunk_ids", []),
+                        child_chunk_ids=child_chunk_ids,
                         chunk_depth=metadata.get("chunk_depth", 0),
-                        decorators=metadata.get("decorators", []),
-                        parameters=metadata.get("parameters", []),
+                        decorators=decorators,
+                        parameters=parameters,
                         return_type=metadata.get("return_type"),
-                        type_annotations=metadata.get("type_annotations", {}),
+                        type_annotations=type_annotations,
                         subproject_name=metadata.get("subproject_name"),
                         subproject_path=metadata.get("subproject_path"),
                     )
@@ -760,6 +791,21 @@ class PooledChromaVectorDatabase(VectorDatabase):
                             "chunk_type": chunk.chunk_type,
                             "function_name": chunk.function_name or "",
                             "class_name": chunk.class_name or "",
+                            "docstring": chunk.docstring or "",
+                            "complexity_score": chunk.complexity_score,
+                            # Hierarchy fields (convert lists to JSON strings for ChromaDB)
+                            "chunk_id": chunk.chunk_id or "",
+                            "parent_chunk_id": chunk.parent_chunk_id or "",
+                            "child_chunk_ids": json.dumps(chunk.child_chunk_ids or []),
+                            "chunk_depth": chunk.chunk_depth,
+                            # Additional metadata (convert lists/dicts to JSON strings)
+                            "decorators": json.dumps(chunk.decorators or []),
+                            "parameters": json.dumps(chunk.parameters or []),
+                            "return_type": chunk.return_type or "",
+                            "type_annotations": json.dumps(chunk.type_annotations or {}),
+                            # Monorepo support
+                            "subproject_name": chunk.subproject_name or "",
+                            "subproject_path": chunk.subproject_path or "",
                         }
                     )
                     ids.append(chunk.id)
@@ -995,6 +1041,23 @@ class PooledChromaVectorDatabase(VectorDatabase):
                         metadata = results["metadatas"][i]
                         content = results["documents"][i]
 
+                        # Parse JSON strings back to lists/dicts
+                        child_chunk_ids = metadata.get("child_chunk_ids", "[]")
+                        if isinstance(child_chunk_ids, str):
+                            child_chunk_ids = json.loads(child_chunk_ids)
+
+                        decorators = metadata.get("decorators", "[]")
+                        if isinstance(decorators, str):
+                            decorators = json.loads(decorators)
+
+                        parameters = metadata.get("parameters", "[]")
+                        if isinstance(parameters, str):
+                            parameters = json.loads(parameters)
+
+                        type_annotations = metadata.get("type_annotations", "{}")
+                        if isinstance(type_annotations, str):
+                            type_annotations = json.loads(type_annotations)
+
                         chunk = CodeChunk(
                             content=content,
                             file_path=Path(metadata["file_path"]),
@@ -1009,12 +1072,12 @@ class PooledChromaVectorDatabase(VectorDatabase):
                             complexity_score=metadata.get("complexity_score", 0.0),
                             chunk_id=metadata.get("chunk_id"),
                             parent_chunk_id=metadata.get("parent_chunk_id"),
-                            child_chunk_ids=metadata.get("child_chunk_ids", []),
+                            child_chunk_ids=child_chunk_ids,
                             chunk_depth=metadata.get("chunk_depth", 0),
-                            decorators=metadata.get("decorators", []),
-                            parameters=metadata.get("parameters", []),
+                            decorators=decorators,
+                            parameters=parameters,
                             return_type=metadata.get("return_type"),
-                            type_annotations=metadata.get("type_annotations", {}),
+                            type_annotations=type_annotations,
                             subproject_name=metadata.get("subproject_name"),
                             subproject_path=metadata.get("subproject_path"),
                         )

mcp_vector_search/core/directory_index.py

@@ -0,0 +1,303 @@
+"""Directory index for tracking project structure and file relationships."""
+
+import json
+from collections import defaultdict
+from pathlib import Path
+from typing import Any
+
+from loguru import logger
+
+from .models import Directory
+
+
+class DirectoryIndex:
+    """Manages directory structure and file-directory relationships."""
+
+    def __init__(self, index_path: Path) -> None:
+        """Initialize directory index.
+
+        Args:
+            index_path: Path to directory index file (JSON)
+        """
+        self.index_path = index_path
+        self.directories: dict[str, Directory] = {}  # path -> Directory
+        self.file_to_directory: dict[str, str] = {}  # file_path -> directory_path
+        self.directory_files: dict[str, list[str]] = defaultdict(list)  # dir_path -> [file_paths]
+
+    def load(self) -> None:
+        """Load directory index from disk."""
+        if not self.index_path.exists():
+            logger.debug("No directory index found, starting fresh")
+            return
+
+        try:
+            with open(self.index_path, "r") as f:
+                data = json.load(f)
+
+            # Load directories
+            for dir_data in data.get("directories", []):
+                directory = Directory.from_dict(dir_data)
+                self.directories[str(directory.path)] = directory
+
+            # Load file mappings
+            self.file_to_directory = data.get("file_to_directory", {})
+
+            # Rebuild directory_files from file_to_directory
+            self.directory_files = defaultdict(list)
+            for file_path, dir_path in self.file_to_directory.items():
+                self.directory_files[dir_path].append(file_path)
+
+            logger.info(f"Loaded {len(self.directories)} directories from index")
+
+        except Exception as e:
+            logger.error(f"Failed to load directory index: {e}")
+            self.directories = {}
+            self.file_to_directory = {}
+            self.directory_files = defaultdict(list)
+
+    def save(self) -> None:
+        """Save directory index to disk."""
+        try:
+            # Ensure parent directory exists
+            self.index_path.parent.mkdir(parents=True, exist_ok=True)
+
+            data = {
+                "directories": [d.to_dict() for d in self.directories.values()],
+                "file_to_directory": self.file_to_directory,
+            }
+
+            with open(self.index_path, "w") as f:
+                json.dump(data, f, indent=2)
+
+            logger.debug(f"Saved {len(self.directories)} directories to index")
+
+        except Exception as e:
+            logger.error(f"Failed to save directory index: {e}")
+            raise
+
+    def add_directory(self, directory: Directory) -> None:
+        """Add or update a directory in the index.
+
+        Args:
+            directory: Directory to add
+        """
+        dir_path = str(directory.path)
+        self.directories[dir_path] = directory
+
+    def add_file(self, file_path: Path, directory_path: Path) -> None:
+        """Associate a file with its directory.
+
+        Args:
+            file_path: Path to the file
+            directory_path: Path to the directory containing the file
+        """
+        file_path_str = str(file_path)
+        dir_path_str = str(directory_path)
+
+        self.file_to_directory[file_path_str] = dir_path_str
+        if file_path_str not in self.directory_files[dir_path_str]:
+            self.directory_files[dir_path_str].append(file_path_str)
+
+        # Update directory file count
+        if dir_path_str in self.directories:
+            self.directories[dir_path_str].file_count = len(
+                self.directory_files[dir_path_str]
+            )
+
+    def get_directory(self, directory_path: Path) -> Directory | None:
+        """Get directory by path.
+
+        Args:
+            directory_path: Path to directory
+
+        Returns:
+            Directory object or None if not found
+        """
+        return self.directories.get(str(directory_path))
+
+    def get_files_in_directory(self, directory_path: Path) -> list[str]:
+        """Get all files in a directory.
+
+        Args:
+            directory_path: Path to directory
+
+        Returns:
+            List of file paths in the directory
+        """
+        return self.directory_files.get(str(directory_path), [])
+
+    def get_subdirectories(self, directory_path: Path) -> list[Directory]:
+        """Get all immediate subdirectories.
+
+        Args:
+            directory_path: Path to parent directory
+
+        Returns:
+            List of subdirectory objects
+        """
+        parent_path_str = str(directory_path)
+        subdirs = []
+
+        for dir_path_str, directory in self.directories.items():
+            if directory.parent_path and str(directory.parent_path) == parent_path_str:
+                subdirs.append(directory)
+
+        return subdirs
+
+    def get_root_directories(self) -> list[Directory]:
+        """Get all root-level directories (no parent).
+
+        Returns:
+            List of root directory objects
+        """
+        return [d for d in self.directories.values() if d.parent_path is None]
+
+    def delete_directory(self, directory_path: Path) -> None:
+        """Remove directory and its file associations.
+
+        Args:
+            directory_path: Path to directory to remove
+        """
+        dir_path_str = str(directory_path)
+
+        # Remove directory
+        if dir_path_str in self.directories:
+            del self.directories[dir_path_str]
+
+        # Remove file associations
+        if dir_path_str in self.directory_files:
+            for file_path in self.directory_files[dir_path_str]:
+                if file_path in self.file_to_directory:
+                    del self.file_to_directory[file_path]
+            del self.directory_files[dir_path_str]
+
+    def delete_file(self, file_path: Path) -> None:
+        """Remove file from directory associations.
+
+        Args:
+            file_path: Path to file to remove
+        """
+        file_path_str = str(file_path)
+
+        if file_path_str in self.file_to_directory:
+            dir_path = self.file_to_directory[file_path_str]
+            del self.file_to_directory[file_path_str]
+
+            # Remove from directory_files
+            if dir_path in self.directory_files:
+                self.directory_files[dir_path] = [
+                    f for f in self.directory_files[dir_path] if f != file_path_str
+                ]
+
+                # Update directory file count
+                if dir_path in self.directories:
+                    self.directories[dir_path].file_count = len(
+                        self.directory_files[dir_path]
+                    )
+
+    def rebuild_from_files(self, file_paths: list[Path], root_path: Path, chunk_stats: dict[str, dict] | None = None) -> None:
+        """Rebuild directory index from list of files with statistics from chunks.
+
+        Args:
+            file_paths: List of file paths to index
+            root_path: Project root path
+            chunk_stats: Optional dict mapping file_path -> {'chunks': count, 'language': str}
+        """
+        self.directories = {}
+        self.file_to_directory = {}
+        self.directory_files = defaultdict(list)
+
+        # Track all unique directories and their statistics
+        dir_set = set()
+        dir_chunks = defaultdict(int)  # directory -> total chunks
+        dir_languages = defaultdict(lambda: defaultdict(int))  # directory -> {language: count}
+        dir_modified = defaultdict(float)  # directory -> most recent modification time
+
+        for file_path in file_paths:
+            # Get relative path from root
+            try:
+                rel_path = file_path.relative_to(root_path)
+                parent_dir = rel_path.parent
+
+                # Add all parent directories up to root
+                current = parent_dir
+                while current != Path("."):
+                    dir_set.add(current)
+
+                    # Accumulate statistics up the directory tree
+                    if chunk_stats and str(file_path) in chunk_stats:
+                        stats = chunk_stats[str(file_path)]
+                        dir_chunks[current] += stats.get('chunks', 0)
+                        if 'language' in stats:
+                            dir_languages[current][stats['language']] += stats.get('chunks', 0)
+                        # Track most recent modification time
+                        if 'modified' in stats:
+                            dir_modified[current] = max(dir_modified.get(current, 0), stats['modified'])
+
+                    current = current.parent
+
+                # Associate file with its direct parent
+                if parent_dir != Path("."):
+                    self.add_file(rel_path, parent_dir)
+
+            except ValueError:
+                # File not relative to root, skip
+                logger.warning(f"File {file_path} not under root {root_path}")
+                continue
+
+        # Create Directory objects for all directories
+        for dir_path in sorted(dir_set):
+            # Determine parent
+            parent_path = dir_path.parent if dir_path.parent != Path(".") else None
+
+            # Check if it's a package
+            is_package = False
+            full_dir_path = root_path / dir_path
+            if (full_dir_path / "__init__.py").exists():
+                is_package = True
+            elif (full_dir_path / "package.json").exists():
+                is_package = True
+
+            directory = Directory(
+                path=dir_path,
+                name=dir_path.name,
+                parent_path=parent_path,
+                depth=len(dir_path.parts),
+                is_package=is_package,
+                total_chunks=dir_chunks.get(dir_path, 0),
+                languages=dict(dir_languages.get(dir_path, {})),
+                last_modified=dir_modified.get(dir_path),
+            )
+
+            self.add_directory(directory)
+
+        # Update subdirectory counts
+        for directory in self.directories.values():
+            subdirs = self.get_subdirectories(directory.path)
+            directory.subdirectory_count = len(subdirs)
+
+        logger.info(f"Rebuilt directory index with {len(self.directories)} directories, {sum(dir_chunks.values())} total chunks")
+
+    def get_stats(self) -> dict[str, Any]:
+        """Get directory index statistics.
+
+        Returns:
+            Dictionary with statistics
+        """
+        return {
+            "total_directories": len(self.directories),
+            "total_files": len(self.file_to_directory),
+            "root_directories": len(self.get_root_directories()),
+            "packages": sum(1 for d in self.directories.values() if d.is_package),
+        }
+
+    def reset(self) -> None:
+        """Clear all directory data."""
+        self.directories = {}
+        self.file_to_directory = {}
+        self.directory_files = defaultdict(list)
+
+        if self.index_path.exists():
+            self.index_path.unlink()
+
+        logger.info("Directory index reset")

mcp_vector_search/core/indexer.py

@@ -15,6 +15,7 @@ from ..parsers.registry import get_parser_registry
 from ..utils.gitignore import create_gitignore_parser
 from ..utils.monorepo import MonorepoDetector
 from .database import VectorDatabase
+from .directory_index import DirectoryIndex
 from .exceptions import ParsingError
 from .models import CodeChunk
 
@@ -29,6 +30,7 @@ class SemanticIndexer:
         file_extensions: list[str],
         max_workers: int | None = None,
         batch_size: int = 10,
+        debug: bool = False,
     ) -> None:
         """Initialize semantic indexer.
 
@@ -38,12 +40,14 @@ class SemanticIndexer:
             file_extensions: File extensions to index
             max_workers: Maximum number of worker threads for parallel processing
             batch_size: Number of files to process in each batch
+            debug: Enable debug output for hierarchy building
         """
         self.database = database
         self.project_root = project_root
         self.file_extensions = {ext.lower() for ext in file_extensions}
         self.parser_registry = get_parser_registry()
         self._ignore_patterns = set(DEFAULT_IGNORE_PATTERNS)
+        self.debug = debug
 
         # Safely get event loop for max_workers
         try:
@@ -81,6 +85,13 @@ class SemanticIndexer:
             for sp in subprojects:
                 logger.debug(f"  - {sp.name} ({sp.relative_path})")
 
+        # Initialize directory index
+        self.directory_index = DirectoryIndex(
+            project_root / ".mcp-vector-search" / "directory_index.json"
+        )
+        # Load existing directory index
+        self.directory_index.load()
+
     async def index_project(
         self,
         force_reindex: bool = False,
@@ -156,6 +167,38 @@ class SemanticIndexer:
 
             self._save_index_metadata(metadata)
 
+            # Rebuild directory index from successfully indexed files
+            try:
+                logger.debug("Rebuilding directory index...")
+                # We don't have chunk counts here, but we have file modification times
+                # Build a simple stats dict with file mod times for recency tracking
+                chunk_stats = {}
+                for file_path in files_to_index:
+                    try:
+                        mtime = os.path.getmtime(file_path)
+                        # For now, just track modification time
+                        # Chunk counts will be aggregated from the database later if needed
+                        chunk_stats[str(file_path)] = {
+                            'modified': mtime,
+                            'chunks': 1,  # Placeholder - real count from chunks
+                        }
+                    except OSError:
+                        pass
+
+                self.directory_index.rebuild_from_files(
+                    files_to_index, self.project_root, chunk_stats=chunk_stats
+                )
+                self.directory_index.save()
+                dir_stats = self.directory_index.get_stats()
+                logger.info(
+                    f"Directory index updated: {dir_stats['total_directories']} directories, "
+                    f"{dir_stats['total_files']} files"
+                )
+            except Exception as e:
+                logger.error(f"Failed to update directory index: {e}")
+                import traceback
+                logger.debug(traceback.format_exc())
+
         logger.info(
             f"Indexing complete: {indexed_count} files indexed, {failed_count} failed"
         )
@@ -307,6 +350,10 @@ class SemanticIndexer:
             # Build hierarchical relationships between chunks
             chunks_with_hierarchy = self._build_chunk_hierarchy(chunks)
 
+            # Debug: Check if hierarchy was built
+            methods_with_parents = sum(1 for c in chunks_with_hierarchy if c.chunk_type in ("method", "function") and c.parent_chunk_id)
+            logger.debug(f"After hierarchy build: {methods_with_parents}/{len([c for c in chunks_with_hierarchy if c.chunk_type in ('method', 'function')])} methods have parents")
+
             # Add chunks to database
             await self.database.add_chunks(chunks_with_hierarchy)
 
@@ -790,6 +837,15 @@ class SemanticIndexer:
         class_chunks = [c for c in chunks if c.chunk_type in ("class", "interface", "mixin")]
         function_chunks = [c for c in chunks if c.chunk_type in ("function", "method", "constructor")]
 
+        # DEBUG: Print what we have (if debug enabled)
+        if self.debug:
+            import sys
+            print(f"\n[DEBUG] Building hierarchy: {len(module_chunks)} modules, {len(class_chunks)} classes, {len(function_chunks)} functions", file=sys.stderr)
+            if class_chunks:
+                print(f"[DEBUG] Class names: {[c.class_name for c in class_chunks[:5]]}", file=sys.stderr)
+            if function_chunks:
+                print(f"[DEBUG] First 5 functions with class_name: {[(f.function_name, f.class_name) for f in function_chunks[:5]]}", file=sys.stderr)
+
         # Build relationships
         for func in function_chunks:
             if func.class_name:
@@ -803,6 +859,10 @@ class SemanticIndexer:
                     func.chunk_depth = parent_class.chunk_depth + 1
                     if func.chunk_id not in parent_class.child_chunk_ids:
                         parent_class.child_chunk_ids.append(func.chunk_id)
+                    if self.debug:
+                        import sys
+                        print(f"[DEBUG] ✓ Linked '{func.function_name}' to class '{parent_class.class_name}'", file=sys.stderr)
+                    logger.debug(f"Linked method '{func.function_name}' (ID: {func.chunk_id[:8]}) to class '{parent_class.class_name}' (ID: {parent_class.chunk_id[:8]})")
             else:
                 # Top-level function
                 if not func.chunk_depth:
@@ -828,6 +888,13 @@ class SemanticIndexer:
             if not mod.chunk_depth:
                 mod.chunk_depth = 0
 
+        # DEBUG: Print summary
+        if self.debug:
+            import sys
+            funcs_with_parents = sum(1 for f in function_chunks if f.parent_chunk_id)
+            classes_with_parents = sum(1 for c in class_chunks if c.parent_chunk_id)
+            print(f"[DEBUG] Hierarchy built: {funcs_with_parents}/{len(function_chunks)} functions linked, {classes_with_parents}/{len(class_chunks)} classes linked\n", file=sys.stderr)
+
         return chunks
 
     def _write_indexing_run_header(self) -> None:

mcp_vector_search/core/models.py

@@ -206,6 +206,64 @@ class IndexStats(BaseModel):
         }
 
 
+@dataclass
+class Directory:
+    """Represents a directory in the project structure."""
+
+    path: Path  # Relative path from project root
+    name: str  # Directory name
+    parent_path: Path | None = None  # Parent directory path (None for root)
+    file_count: int = 0  # Number of files directly in this directory
+    subdirectory_count: int = 0  # Number of subdirectories
+    total_chunks: int = 0  # Total code chunks in this directory (recursive)
+    languages: dict[str, int] = None  # Language distribution in this directory
+    depth: int = 0  # Depth from project root (0 = root)
+    is_package: bool = False  # True if contains __init__.py or package.json
+    last_modified: float | None = None  # Most recent file modification time (unix timestamp)
+
+    def __post_init__(self) -> None:
+        """Initialize default values and generate directory ID."""
+        if self.languages is None:
+            self.languages = {}
+
+    @property
+    def id(self) -> str:
+        """Generate unique ID for this directory."""
+        import hashlib
+        return hashlib.sha256(str(self.path).encode()).hexdigest()[:16]
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary for storage."""
+        return {
+            "path": str(self.path),
+            "name": self.name,
+            "parent_path": str(self.parent_path) if self.parent_path else None,
+            "file_count": self.file_count,
+            "subdirectory_count": self.subdirectory_count,
+            "total_chunks": self.total_chunks,
+            "languages": self.languages,
+            "depth": self.depth,
+            "is_package": self.is_package,
+            "last_modified": self.last_modified,
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "Directory":
+        """Create from dictionary."""
+        return cls(
+            path=Path(data["path"]),
+            name=data["name"],
+            parent_path=Path(data["parent_path"]) if data.get("parent_path") else None,
+            file_count=data.get("file_count", 0),
+            subdirectory_count=data.get("subdirectory_count", 0),
+            total_chunks=data.get("total_chunks", 0),
+            languages=data.get("languages", {}),
+            depth=data.get("depth", 0),
+            is_package=data.get("is_package", False),
+            last_modified=data.get("last_modified"),
+        )
+
+
 class ProjectInfo(BaseModel):
     """Information about a project."""
 

mcp_vector_search/utils/gitignore.py

@@ -51,14 +51,24 @@ class GitignorePattern:
         Returns:
             True if the pattern matches
         """
-        # Directory-only patterns only match directories
-        if self.is_directory_only and not is_directory:
-            return False
-
         # Convert path separators for consistent matching
         path = path.replace("\\", "/")
         pattern = self.pattern.replace("\\", "/")
 
+        # For directory-only patterns, check if any parent directory matches
+        # This implements Git's behavior where "dir/" excludes both the directory
+        # AND all files within it recursively
+        if self.is_directory_only:
+            path_parts = path.split("/")
+            # Check each parent directory component
+            for i in range(1, len(path_parts) + 1):
+                parent = "/".join(path_parts[:i])
+                if fnmatch.fnmatch(parent, pattern):
+                    return True
+            # If no parent matches and this is not a directory, don't exclude
+            if not is_directory:
+                return False
+
         # Try exact match first
         if fnmatch.fnmatch(path, pattern):
             return True