hanzo-mcp 0.5.1__py3-none-any.whl → 0.6.1__py3-none-any.whl

hanzo_mcp/tools/vector/__init__.py

@@ -9,7 +9,7 @@ Supported backends:
 
 from hanzo_mcp.tools.common.base import BaseTool
 from hanzo_mcp.tools.common.permissions import PermissionManager
-from fastmcp import FastMCP
+from mcp.server import FastMCP
 
 # Try to import vector dependencies
 try:
@@ -17,6 +17,7 @@ try:
     from .project_manager import ProjectVectorManager
     from .vector_index import VectorIndexTool
     from .vector_search import VectorSearchTool
+    from .index_tool import IndexTool
     
     VECTOR_AVAILABLE = True
     
@@ -64,6 +65,9 @@ try:
                 print(f"Detected {len(detected_projects)} projects with LLM.md files")
         
         # Register individual tools if enabled
+        if tool_enabled.get("index", True):
+            tools.append(IndexTool(permission_manager))
+        
         if tool_enabled.get("vector_index", True):
             tools.append(VectorIndexTool(permission_manager, project_manager))
         
@@ -94,6 +98,7 @@ if VECTOR_AVAILABLE:
     __all__.extend([
         "InfinityVectorStore",
         "ProjectVectorManager",
+        "IndexTool",
         "VectorIndexTool", 
         "VectorSearchTool",
     ])

hanzo_mcp/tools/vector/git_ingester.py

@@ -89,9 +89,11 @@ class GitIngester:
             "repository": str(repo_path),
             "branch": branch,
             "commits_processed": 0,
+            "commits_indexed": 0,
             "files_indexed": 0,
             "symbols_extracted": 0,
             "diffs_indexed": 0,
+            "blame_entries": 0,
             "errors": []
         }
         
@@ -125,6 +127,7 @@ class GitIngester:
                 
                 for commit in commits:
                     self._index_commit(commit, include_diffs=include_diffs)
+                    results["commits_indexed"] = results.get("commits_indexed", 0) + 1
                     
                     if include_diffs:
                         results["diffs_indexed"] += len(commit.files)

hanzo_mcp/tools/vector/index_tool.py

@@ -0,0 +1,358 @@
+"""Index tool for managing vector store indexing."""
+
+import asyncio
+import os
+import time
+from pathlib import Path
+from typing import Annotated, TypedDict, Unpack, final, override
+
+from mcp.server.fastmcp import Context as MCPContext
+from pydantic import Field
+
+from hanzo_mcp.tools.common.base import BaseTool
+from hanzo_mcp.tools.common.context import create_tool_context
+from hanzo_mcp.tools.common.permissions import PermissionManager
+from hanzo_mcp.tools.vector.git_ingester import GitIngester
+from hanzo_mcp.tools.vector.infinity_store import InfinityVectorStore
+from hanzo_mcp.tools.vector.project_manager import ProjectVectorManager
+
+
+Path_str = Annotated[
+    str,
+    Field(
+        description="Path to index (defaults to current working directory)",
+        min_length=1,
+    ),
+]
+
+IncludeGitHistory = Annotated[
+    bool,
+    Field(
+        description="Include git history in the index",
+        default=True,
+    ),
+]
+
+FilePatterns = Annotated[
+    list[str] | None,
+    Field(
+        description="File patterns to include (e.g., ['*.py', '*.js'])",
+        default=None,
+    ),
+]
+
+ShowStats = Annotated[
+    bool,
+    Field(
+        description="Show detailed statistics after indexing",
+        default=True,
+    ),
+]
+
+Force = Annotated[
+    bool,
+    Field(
+        description="Force re-indexing even if already indexed",
+        default=False,
+    ),
+]
+
+
+class IndexToolParams(TypedDict, total=False):
+    """Parameters for the index tool."""
+
+    path: str
+    include_git_history: bool
+    file_patterns: list[str] | None
+    show_stats: bool
+    force: bool
+
+
+@final
+class IndexTool(BaseTool):
+    """Tool for indexing files and git history into vector store."""
+
+    def __init__(self, permission_manager: PermissionManager):
+        """Initialize the index tool.
+
+        Args:
+            permission_manager: Permission manager for access control
+        """
+        self.permission_manager = permission_manager
+        self.project_manager = ProjectVectorManager(permission_manager)
+
+    @property
+    @override
+    def name(self) -> str:
+        """Get the tool name."""
+        return "index"
+
+    @property
+    @override
+    def description(self) -> str:
+        """Get the tool description."""
+        return """Index files and git history into the vector store for semantic search.
+
+This tool:
+- Indexes all project files into a vector database
+- Includes git history (commits, diffs, blame) when available
+- Supports incremental updates
+- Shows statistics about indexed content
+- Automatically creates project-specific databases
+
+Usage:
+- index: Index the current directory
+- index --path /path/to/project: Index a specific path
+- index --file-patterns "*.py" "*.js": Index only specific file types
+- index --no-git-history: Skip git history indexing
+- index --force: Force re-indexing of all files"""
+
+    @override
+    async def call(
+        self,
+        ctx: MCPContext,
+        **params: Unpack[IndexToolParams],
+    ) -> str:
+        """Execute the index tool.
+
+        Args:
+            ctx: MCP context
+            **params: Tool parameters
+
+        Returns:
+            Indexing result and statistics
+        """
+        start_time = time.time()
+        tool_ctx = create_tool_context(ctx)
+        await tool_ctx.set_tool_info(self.name)
+
+        # Extract parameters
+        path = params.get("path", os.getcwd())
+        include_git_history = params.get("include_git_history", True)
+        file_patterns = params.get("file_patterns")
+        show_stats = params.get("show_stats", True)
+        force = params.get("force", False)
+
+        # Resolve absolute path
+        abs_path = os.path.abspath(path)
+        
+        # Check permissions
+        if not self.permission_manager.has_permission(abs_path):
+            return f"Permission denied: {abs_path}"
+
+        # Check if path exists
+        if not os.path.exists(abs_path):
+            return f"Path does not exist: {abs_path}"
+
+        await tool_ctx.info(f"Starting indexing of {abs_path}")
+
+        try:
+            # Get or create vector store for this project
+            vector_store = self.project_manager.get_project_store(abs_path)
+            
+            # Check if already indexed (unless force)
+            if not force:
+                stats = await vector_store.get_stats()
+                if stats and stats.get("document_count", 0) > 0:
+                    await tool_ctx.info("Project already indexed, use --force to re-index")
+                    if show_stats:
+                        return self._format_stats(stats, abs_path, time.time() - start_time)
+                    return "Project is already indexed. Use --force to re-index."
+
+            # Prepare file patterns
+            if file_patterns is None:
+                # Default patterns for code files
+                file_patterns = [
+                    "*.py", "*.js", "*.ts", "*.jsx", "*.tsx",
+                    "*.java", "*.cpp", "*.c", "*.h", "*.hpp",
+                    "*.go", "*.rs", "*.rb", "*.php", "*.swift",
+                    "*.kt", "*.scala", "*.cs", "*.vb", "*.fs",
+                    "*.sh", "*.bash", "*.zsh", "*.fish",
+                    "*.md", "*.rst", "*.txt", "*.json", "*.yaml", "*.yml",
+                    "*.toml", "*.ini", "*.cfg", "*.conf",
+                    "*.html", "*.css", "*.scss", "*.sass", "*.less",
+                    "*.sql", "*.graphql", "*.proto",
+                    "Dockerfile", "Makefile", "*.mk",
+                    ".gitignore", ".dockerignore", "requirements.txt",
+                    "package.json", "Cargo.toml", "go.mod", "pom.xml",
+                ]
+
+            # Clear existing index if force
+            if force:
+                await tool_ctx.info("Clearing existing index...")
+                await vector_store.clear()
+
+            # Index files
+            await tool_ctx.info("Indexing files...")
+            indexed_files = 0
+            total_size = 0
+            errors = []
+
+            for pattern in file_patterns:
+                pattern_files = await self._find_files(abs_path, pattern)
+                for file_path in pattern_files:
+                    try:
+                        # Check file size (skip very large files)
+                        file_size = os.path.getsize(file_path)
+                        if file_size > 10 * 1024 * 1024:  # 10MB
+                            await tool_ctx.warning(f"Skipping large file: {file_path}")
+                            continue
+
+                        # Read file content
+                        try:
+                            with open(file_path, "r", encoding="utf-8") as f:
+                                content = f.read()
+                        except UnicodeDecodeError:
+                            # Skip binary files
+                            continue
+
+                        # Index the file
+                        rel_path = os.path.relpath(file_path, abs_path)
+                        await vector_store.index_document(
+                            content=content,
+                            metadata={
+                                "type": "file",
+                                "path": rel_path,
+                                "absolute_path": file_path,
+                                "size": file_size,
+                                "extension": Path(file_path).suffix,
+                            }
+                        )
+                        indexed_files += 1
+                        total_size += file_size
+
+                        if indexed_files % 100 == 0:
+                            await tool_ctx.info(f"Indexed {indexed_files} files...")
+
+                    except Exception as e:
+                        errors.append(f"{file_path}: {str(e)}")
+
+            await tool_ctx.info(f"Indexed {indexed_files} files ({total_size / 1024 / 1024:.1f} MB)")
+
+            # Index git history if requested
+            git_stats = {}
+            if include_git_history and os.path.exists(os.path.join(abs_path, ".git")):
+                await tool_ctx.info("Indexing git history...")
+                
+                git_ingester = GitIngester(vector_store)
+                git_stats = await git_ingester.ingest_repository(
+                    repo_path=abs_path,
+                    include_history=True,
+                    include_diffs=True,
+                    include_blame=True,
+                    file_patterns=file_patterns,
+                )
+                
+                await tool_ctx.info(
+                    f"Indexed {git_stats.get('commits_indexed', 0)} commits, "
+                    f"{git_stats.get('diffs_indexed', 0)} diffs"
+                )
+
+            # Get final statistics
+            if show_stats:
+                stats = await vector_store.get_stats()
+                stats.update({
+                    "files_indexed": indexed_files,
+                    "total_size_mb": total_size / 1024 / 1024,
+                    "errors": len(errors),
+                    **git_stats,
+                })
+                result = self._format_stats(stats, abs_path, time.time() - start_time)
+                
+                if errors:
+                    result += f"\n\nErrors ({len(errors)}):\n"
+                    result += "\n".join(errors[:10])  # Show first 10 errors
+                    if len(errors) > 10:
+                        result += f"\n... and {len(errors) - 10} more errors"
+                
+                return result
+            else:
+                return f"Successfully indexed {indexed_files} files"
+
+        except Exception as e:
+            await tool_ctx.error(f"Indexing failed: {str(e)}")
+            return f"Error during indexing: {str(e)}"
+
+    async def _find_files(self, base_path: str, pattern: str) -> list[str]:
+        """Find files matching a pattern.
+
+        Args:
+            base_path: Base directory to search
+            pattern: File pattern to match
+
+        Returns:
+            List of matching file paths
+        """
+        import glob
+
+        # Use glob to find files
+        if pattern.startswith("*."):
+            # Extension pattern
+            files = glob.glob(
+                os.path.join(base_path, "**", pattern),
+                recursive=True,
+            )
+        else:
+            # Exact filename
+            files = glob.glob(
+                os.path.join(base_path, "**", pattern),
+                recursive=True,
+            )
+
+        # Filter out hidden directories and common ignore patterns
+        filtered_files = []
+        ignore_dirs = {".git", "__pycache__", "node_modules", ".venv", "venv", "dist", "build"}
+        
+        for file_path in files:
+            # Check if any parent directory is in ignore list
+            parts = Path(file_path).parts
+            if any(part in ignore_dirs for part in parts):
+                continue
+            if any(part.startswith(".") and part != "." for part in parts[:-1]):
+                continue  # Skip hidden directories (but allow hidden files like .gitignore)
+            filtered_files.append(file_path)
+
+        return filtered_files
+
+    def _format_stats(self, stats: dict, path: str, elapsed_time: float) -> str:
+        """Format statistics for display.
+
+        Args:
+            stats: Statistics dictionary
+            path: Indexed path
+            elapsed_time: Time taken for indexing
+
+        Returns:
+            Formatted statistics string
+        """
+        result = f"=== Index Statistics for {path} ===\n\n"
+        
+        # Basic stats
+        result += f"Indexing completed in {elapsed_time:.1f} seconds\n\n"
+        
+        result += "Content Statistics:\n"
+        result += f"  Documents: {stats.get('document_count', 0):,}\n"
+        result += f"  Files indexed: {stats.get('files_indexed', 0):,}\n"
+        result += f"  Total size: {stats.get('total_size_mb', 0):.1f} MB\n"
+        
+        if stats.get("commits_indexed", 0) > 0:
+            result += f"\nGit History:\n"
+            result += f"  Commits: {stats.get('commits_indexed', 0):,}\n"
+            result += f"  Diffs: {stats.get('diffs_indexed', 0):,}\n"
+            result += f"  Blame entries: {stats.get('blame_entries', 0):,}\n"
+        
+        # Vector store info
+        result += f"\nVector Store:\n"
+        result += f"  Database: {stats.get('database_name', 'default')}\n"
+        result += f"  Table: {stats.get('table_name', 'documents')}\n"
+        result += f"  Vectors: {stats.get('vector_count', stats.get('document_count', 0)):,}\n"
+        
+        if stats.get("errors", 0) > 0:
+            result += f"\nErrors: {stats.get('errors', 0)}\n"
+        
+        return result
+
+    def register(self, mcp_server) -> None:
+        """Register this tool with the MCP server."""
+        # Tool registration is handled by the ToolRegistry
+        pass

hanzo_mcp/tools/vector/infinity_store.py

@@ -725,6 +725,104 @@ class InfinityVectorStore:
         import random
         return [random.random() for _ in range(self.dimension)]
     
+    async def get_stats(self) -> Dict[str, Any]:
+        """Get statistics about the vector store.
+        
+        Returns:
+            Dictionary with statistics
+        """
+        try:
+            # Get document count
+            doc_count_result = self.documents_table.output(["count(*)"]).to_pl()
+            doc_count = doc_count_result.item(0, 0) if len(doc_count_result) > 0 else 0
+            
+            # Get unique file count
+            file_result = self.documents_table.output(["file_path"]).to_pl()
+            unique_files = set()
+            for row in file_result.iter_rows():
+                if row[0]:
+                    unique_files.add(row[0])
+            
+            # Get symbol count
+            symbol_count = 0
+            try:
+                symbol_result = self.symbols_table.output(["count(*)"]).to_pl()
+                symbol_count = symbol_result.item(0, 0) if len(symbol_result) > 0 else 0
+            except:
+                pass
+            
+            # Get AST count
+            ast_count = 0
+            try:
+                ast_result = self.ast_table.output(["count(*)"]).to_pl()
+                ast_count = ast_result.item(0, 0) if len(ast_result) > 0 else 0
+            except:
+                pass
+            
+            return {
+                "document_count": doc_count,
+                "vector_count": doc_count,  # Each document has a vector
+                "unique_files": len(unique_files),
+                "symbol_count": symbol_count,
+                "ast_count": ast_count,
+                "database_name": self.db_name,
+                "table_name": "documents",
+                "dimension": self.dimension,
+            }
+        except Exception as e:
+            return {
+                "error": str(e),
+                "document_count": 0,
+                "vector_count": 0,
+            }
+    
+    async def clear(self) -> bool:
+        """Clear all data from the vector store.
+        
+        Returns:
+            True if successful
+        """
+        try:
+            # Delete all records from all tables
+            self.documents_table.delete()
+            
+            try:
+                self.symbols_table.delete()
+            except:
+                pass
+            
+            try:
+                self.ast_table.delete()
+            except:
+                pass
+            
+            try:
+                self.references_table.delete()
+            except:
+                pass
+            
+            return True
+        except Exception as e:
+            print(f"Error clearing vector store: {e}")
+            return False
+    
+    async def index_document(
+        self,
+        content: str,
+        metadata: Dict[str, Any] = None,
+    ) -> str:
+        """Async version of add_document for consistency.
+        
+        Args:
+            content: Document content
+            metadata: Additional metadata
+            
+        Returns:
+            Document ID
+        """
+        file_path = metadata.get("path") if metadata else None
+        return self.add_document(content, metadata, file_path)
+    
     def close(self):
         """Close the database connection."""
         if hasattr(self, 'infinity'):

hanzo_mcp/tools/vector/project_manager.py

@@ -8,6 +8,7 @@ import asyncio
 from concurrent.futures import ThreadPoolExecutor
 
 from .infinity_store import InfinityVectorStore, SearchResult
+from hanzo_mcp.tools.config.index_config import IndexConfig, IndexScope
 
 
 @dataclass
@@ -38,12 +39,14 @@ class ProjectVectorManager:
         self.embedding_model = embedding_model
         self.dimension = dimension
         
+        # Set up index configuration
+        self.index_config = IndexConfig()
+        
         # Set up global database path
         if global_db_path:
             self.global_db_path = Path(global_db_path)
         else:
-            from hanzo_mcp.config.settings import get_config_dir
-            self.global_db_path = get_config_dir() / "db"
+            self.global_db_path = self.index_config.get_index_path("vector")
         
         self.global_db_path.mkdir(parents=True, exist_ok=True)
         
@@ -158,14 +161,25 @@ class ProjectVectorManager:
         Returns:
             Vector store instance
         """
-        if project_info is None:
+        # Check indexing scope
+        if project_info:
+            scope = self.index_config.get_scope(str(project_info.root_path))
+            if scope == IndexScope.GLOBAL:
+                # Even for project files, use global store if configured
+                return self._get_global_store()
+        else:
             return self._get_global_store()
         
+        # Use project-specific store
         project_key = str(project_info.root_path)
         
         if project_key not in self.vector_stores:
+            # Get index path based on configuration
+            index_path = self.index_config.get_index_path("vector", str(project_info.root_path))
+            index_path.mkdir(parents=True, exist_ok=True)
+            
             self.vector_stores[project_key] = InfinityVectorStore(
-                data_path=str(project_info.db_path),
+                data_path=str(index_path),
                 embedding_model=self.embedding_model,
                 dimension=self.dimension,
             )
@@ -190,10 +204,14 @@ class ProjectVectorManager:
         Returns:
             Tuple of (document IDs, project info or None for global)
         """
+        # Check if indexing is enabled
+        if not self.index_config.is_indexing_enabled("vector"):
+            return [], None
+            
         # Find project for this file
         project_info = self.get_project_for_path(file_path)
         
-        # Get appropriate vector store
+        # Get appropriate vector store based on scope configuration
         vector_store = self.get_vector_store(project_info)
         
         # Add file metadata
@@ -201,8 +219,12 @@ class ProjectVectorManager:
         if project_info:
             file_metadata["project_name"] = project_info.name
             file_metadata["project_root"] = str(project_info.root_path)
+            # Check actual scope used
+            scope = self.index_config.get_scope(str(project_info.root_path))
+            file_metadata["index_scope"] = scope.value
         else:
             file_metadata["project_name"] = "global"
+            file_metadata["index_scope"] = "global"
         
         # Add file to store
         doc_ids = vector_store.add_file(