mcp-code-indexer 4.0.1__py3-none-any.whl → 4.1.0__py3-none-any.whl

mcp_code_indexer/git_hook_handler.py

@@ -10,17 +10,16 @@ import asyncio
 import json
 import logging
 import os
-import subprocess
+import subprocess  # nosec B404
 from pathlib import Path
-from typing import Dict, List, Optional, Tuple, Any
-
+from typing import Any, Dict, List, Optional, Tuple, Union, cast
 
 import aiohttp
 from tenacity import (
     retry,
-    wait_exponential,
-    stop_after_attempt,
     retry_if_exception_type,
+    stop_after_attempt,
+    wait_exponential,
 )
 
 from .database.database import DatabaseManager
@@ -73,7 +72,7 @@ class GitHookHandler:
         self.token_counter = TokenCounter()
 
         # Git hook specific settings
-        self.config = {
+        self.config: Dict[str, Union[str, int, float]] = {
             "model": os.getenv("MCP_GITHOOK_MODEL", self.OPENROUTER_MODEL),
             "max_diff_tokens": 136000,  # Skip if diff larger than this (in tokens)
             "chunk_token_limit": 100000,  # Target token limit per chunk
@@ -91,14 +90,14 @@ class GitHookHandler:
     def _log_and_print(self, message: str, level: str = "info") -> None:
         """
         Log message and also print to stdout for user visibility.
-        
+
         Args:
             message: Message to log and print
             level: Log level (info, warning, error)
         """
         # Log to logger
         getattr(self.logger, level)(message)
-        
+
         # Also print to stdout with prefix for visibility
         prefix = "🔍" if level == "info" else "⚠️" if level == "warning" else "❌"
         print(f"{prefix} {message}")
@@ -130,9 +129,7 @@ class GitHookHandler:
 
             # Get git info from current directory
             project_info = await self._identify_project_from_git()
-            self._log_and_print(
-                f"Project: {project_info.get('name', 'Unknown')}"
-            )
+            self._log_and_print(f"Project: {project_info.get('name', 'Unknown')}")
 
             # Get git diff and commit message based on mode
             if commit_hash:
@@ -187,11 +184,11 @@ class GitHookHandler:
 
             # Apply updates to database
             await self._apply_updates(project_info, updates)
-            
+
             # Count actual updates
             file_update_count = len(updates.get("file_updates", {}))
             overview_updated = bool(updates.get("overview_update"))
-            
+
             if file_update_count > 0 or overview_updated:
                 update_parts = []
                 if file_update_count > 0:
@@ -250,7 +247,7 @@ class GitHookHandler:
         self.logger.info(f"Single-stage prompt: {prompt_tokens} tokens")
         self.logger.info(f"Token limit: {token_limit}")
 
-        if prompt_tokens <= token_limit:
+        if prompt_tokens <= int(token_limit):
             # Use single-stage approach
             self._log_and_print("Using single-stage analysis")
             result = await self._call_openrouter(single_stage_prompt)
@@ -262,18 +259,22 @@ class GitHookHandler:
             # Try two-stage analysis first
             try:
                 return await self._analyze_with_two_stage(
-                    git_diff, commit_message, current_overview,
-                    current_descriptions, changed_files
+                    git_diff,
+                    commit_message,
+                    current_overview,
+                    current_descriptions,
+                    changed_files,
                 )
             except GitHookError as e:
                 if "too large" in str(e).lower():
                     # Fall back to chunked processing
-                    self._log_and_print(
-                        "Using chunked processing (very large diff)"
-                    )
+                    self._log_and_print("Using chunked processing (very large diff)")
                     return await self._analyze_with_chunking(
-                        git_diff, commit_message, current_overview,
-                        current_descriptions, changed_files
+                        git_diff,
+                        commit_message,
+                        current_overview,
+                        current_descriptions,
+                        changed_files,
                     )
                 else:
                     raise
@@ -319,7 +320,7 @@ CURRENT FILE DESCRIPTIONS (for changed files only):
 {json.dumps(relevant_descriptions, indent=2)}
 
 CHANGED FILES:
-{', '.join(changed_files)}
+{", ".join(changed_files)}
 
 GIT DIFF:
 {git_diff}
@@ -663,44 +664,39 @@ Return ONLY a JSON object:
             )
 
         # Break changed files into chunks and process file descriptions
-        chunk_size = await self._calculate_optimal_chunk_size(
-            git_diff, changed_files
-        )
-        
+        chunk_size = await self._calculate_optimal_chunk_size(git_diff, changed_files)
+
         self._log_and_print(f"Processing in {chunk_size}-file chunks")
-        
+
         all_file_updates = {}
-        
+
         for i in range(0, len(changed_files), chunk_size):
-            chunk_files = changed_files[i:i + chunk_size]
+            chunk_files = changed_files[i : i + chunk_size]
             chunk_number = (i // chunk_size) + 1
             total_chunks = (len(changed_files) + chunk_size - 1) // chunk_size
-            
+
             self._log_and_print(
                 f"Processing chunk {chunk_number}/{total_chunks} "
                 f"({len(chunk_files)} files)"
             )
-            
+
             # Extract diff content for this chunk
             chunk_diff = self._extract_chunk_diff(git_diff, chunk_files)
-            
+
             # Process this chunk
             chunk_updates = await self._analyze_file_chunk(
                 chunk_diff, commit_message, current_descriptions, chunk_files
             )
-            
+
             # Merge results
             if chunk_updates and "file_updates" in chunk_updates:
                 all_file_updates.update(chunk_updates["file_updates"])
-        
+
         self.logger.info(
             f"Chunked processing completed: updated {len(all_file_updates)} files"
         )
-        
-        return {
-            "file_updates": all_file_updates,
-            "overview_update": overview_update
-        }
+
+        return {"file_updates": all_file_updates, "overview_update": overview_update}
 
     async def _analyze_overview_updates(
         self,
@@ -733,7 +729,7 @@ CURRENT PROJECT OVERVIEW:
 {current_overview or "No overview available"}
 
 CHANGED FILES:
-{', '.join(changed_files)}
+{", ".join(changed_files)}
 
 GIT DIFF:
 {git_diff}
@@ -762,10 +758,8 @@ Return ONLY a JSON object:
         prompt_tokens = self.token_counter.count_tokens(prompt)
         self.logger.info(f"Stage 1 prompt: {prompt_tokens} tokens")
 
-        if prompt_tokens > self.config["max_diff_tokens"]:
-            raise GitHookError(
-                f"Stage 1 prompt too large ({prompt_tokens} tokens)"
-            )
+        if prompt_tokens > int(self.config["max_diff_tokens"]):
+            raise GitHookError(f"Stage 1 prompt too large ({prompt_tokens} tokens)")
 
         # Call OpenRouter API
         result = await self._call_openrouter(prompt)
@@ -811,7 +805,7 @@ CURRENT FILE DESCRIPTIONS (for changed files only):
 {json.dumps(relevant_descriptions, indent=2)}
 
 CHANGED FILES:
-{', '.join(changed_files)}
+{", ".join(changed_files)}
 
 GIT DIFF:
 {git_diff}
@@ -836,10 +830,8 @@ Return ONLY a JSON object:
         prompt_tokens = self.token_counter.count_tokens(prompt)
         self.logger.info(f"Stage 2 prompt: {prompt_tokens} tokens")
 
-        if prompt_tokens > self.config["max_diff_tokens"]:
-            raise GitHookError(
-                f"Stage 2 prompt too large ({prompt_tokens} tokens)"
-            )
+        if prompt_tokens > int(self.config["max_diff_tokens"]):
+            raise GitHookError(f"Stage 2 prompt too large ({prompt_tokens} tokens)")
 
         # Call OpenRouter API
         result = await self._call_openrouter(prompt)
@@ -875,7 +867,7 @@ CURRENT PROJECT OVERVIEW:
 {current_overview or "No overview available"}
 
 CHANGED FILES:
-{', '.join(changed_files)}
+{", ".join(changed_files)}
 
 INSTRUCTIONS:
 Update project overview ONLY if there are major structural changes like:
@@ -918,26 +910,26 @@ Return ONLY a JSON object:
         # Estimate average diff size per file
         total_diff_tokens = self.token_counter.count_tokens(git_diff)
         avg_tokens_per_file = total_diff_tokens / len(changed_files)
-        
+
         # Target chunk token limit
         chunk_limit = self.config.get("chunk_token_limit", 100000)
-        
+
         # Calculate chunk size with buffer for overhead
         overhead_factor = 0.7  # Reserve 30% for prompt overhead
-        effective_limit = chunk_limit * overhead_factor
-        
+        effective_limit = int(chunk_limit) * overhead_factor
+
         chunk_size = max(1, int(effective_limit / avg_tokens_per_file))
-        
+
         # Cap at reasonable limits
         chunk_size = min(chunk_size, 50)  # Max 50 files per chunk
-        chunk_size = max(chunk_size, 5)   # Min 5 files per chunk
-        
+        chunk_size = max(chunk_size, 5)  # Min 5 files per chunk
+
         self.logger.info(
             f"Calculated chunk size: {chunk_size} files "
             f"(avg {avg_tokens_per_file:.0f} tokens/file, "
             f"target {chunk_limit} tokens/chunk)"
         )
-        
+
         return chunk_size
 
     def _extract_chunk_diff(self, git_diff: str, chunk_files: List[str]) -> str:
@@ -951,24 +943,22 @@ Return ONLY a JSON object:
         Returns:
             Filtered diff content for chunk files only
         """
-        lines = git_diff.split('\n')
+        lines = git_diff.split("\n")
         chunk_lines = []
-        current_file = None
         include_section = False
-        
+
         for line in lines:
-            if line.startswith('diff --git'):
+            if line.startswith("diff --git"):
                 # Parse file path from diff header
-                parts = line.split(' ')
+                parts = line.split(" ")
                 if len(parts) >= 4:
                     file_path = parts[2][2:]  # Remove 'a/' prefix
-                    current_file = file_path
                     include_section = file_path in chunk_files
-            
+
             if include_section:
                 chunk_lines.append(line)
-        
-        return '\n'.join(chunk_lines)
+
+        return "\n".join(chunk_lines)
 
     async def _analyze_file_chunk(
         self,
@@ -1005,14 +995,14 @@ CURRENT FILE DESCRIPTIONS (for chunk files only):
 {json.dumps(relevant_descriptions, indent=2)}
 
 CHUNK FILES:
-{', '.join(chunk_files)}
+{", ".join(chunk_files)}
 
 GIT DIFF (chunk only):
 {chunk_diff}
 
 INSTRUCTIONS:
 Use the COMMIT MESSAGE to understand the intent and context of the changes.
-Update descriptions for files that have changed significantly. 
+Update descriptions for files that have changed significantly.
 Only include files that need actual description updates.
 
 Return ONLY a JSON object:
@@ -1027,7 +1017,7 @@ Return ONLY a JSON object:
         prompt_tokens = self.token_counter.count_tokens(prompt)
         self.logger.info(f"Chunk prompt: {prompt_tokens} tokens")
 
-        if prompt_tokens > self.config.get("chunk_token_limit", 100000):
+        if prompt_tokens > int(self.config.get("chunk_token_limit", 100000)):
             self.logger.warning(
                 f"Chunk still too large ({prompt_tokens} tokens), "
                 f"skipping {len(chunk_files)} files"
@@ -1081,7 +1071,7 @@ Return ONLY a JSON object:
             "max_tokens": 24000,
         }
 
-        timeout = aiohttp.ClientTimeout(total=self.config["timeout"])
+        timeout = aiohttp.ClientTimeout(total=float(self.config["timeout"]))
 
         self.logger.info("Sending request to OpenRouter API...")
         self.logger.info(f"  Model: {self.config['model']}")
@@ -1094,7 +1084,6 @@ Return ONLY a JSON object:
                 async with session.post(
                     self.OPENROUTER_API_URL, headers=headers, json=payload
                 ) as response:
-
                     self.logger.info(
                         f"OpenRouter API response status: {response.status}"
                     )
@@ -1221,7 +1210,7 @@ Return ONLY a JSON object:
                     "Response must contain 'file_updates' and/or 'overview_update'"
                 )
 
-            return data
+            return cast(Dict[str, Any], data)
 
         except json.JSONDecodeError as e:
             self.logger.error(f"Raw response content: {repr(response_text)}")
@@ -1249,11 +1238,15 @@ Return ONLY a JSON object:
             # Update file descriptions
             file_updates = updates.get("file_updates", {})
             for file_path, description in file_updates.items():
-                from mcp_code_indexer.database.models import FileDescription
                 from datetime import datetime
 
+                from mcp_code_indexer.database.models import FileDescription
+
                 file_desc = FileDescription(
+                    id=None,
                     project_id=project.id,
+                    source_project_id=None,
+                    to_be_cleaned=None,
                     file_path=file_path,
                     description=description,
                     file_hash=None,
@@ -1266,9 +1259,10 @@ Return ONLY a JSON object:
             # Update project overview if provided
             overview_update = updates.get("overview_update")
             if overview_update and overview_update.strip():
-                from mcp_code_indexer.database.models import ProjectOverview
                 from datetime import datetime
 
+                from mcp_code_indexer.database.models import ProjectOverview
+
                 overview = ProjectOverview(
                     project_id=project.id,
                     overview=overview_update,
@@ -1305,8 +1299,9 @@ Return ONLY a JSON object:
             stdout, stderr = await process.communicate()
 
             if process.returncode != 0:
+                returncode = process.returncode if process.returncode is not None else 1
                 raise subprocess.CalledProcessError(
-                    process.returncode, full_cmd, stdout, stderr
+                    returncode, full_cmd, stdout, stderr
                 )
 
             return stdout.decode("utf-8")

mcp_code_indexer/logging_config.py

@@ -9,7 +9,7 @@ import logging
 import logging.handlers
 import sys
 from pathlib import Path
-from typing import Optional
+from typing import Optional, Union
 
 from .error_handler import StructuredFormatter
 
@@ -58,6 +58,9 @@ def setup_logging(
             log_file.parent.mkdir(parents=True, exist_ok=True)
 
             # Rotating file handler
+            file_handler: Union[
+                logging.handlers.RotatingFileHandler, logging.FileHandler
+            ]
             if max_bytes > 0:
                 file_handler = logging.handlers.RotatingFileHandler(
                     log_file,
@@ -144,7 +147,10 @@ def setup_command_logger(
 
         # File handler with 50MB limit
         file_handler = logging.handlers.RotatingFileHandler(
-            log_file, maxBytes=50 * 1024 * 1024, backupCount=2, encoding="utf-8"  # 50MB
+            log_file,
+            maxBytes=50 * 1024 * 1024,
+            backupCount=2,
+            encoding="utf-8",  # 50MB
         )
         file_handler.setLevel(logging.DEBUG)
 
@@ -172,7 +178,9 @@ def setup_command_logger(
 
 
 def _setup_component_loggers_for_command(
-    command_name: str, file_handler: logging.Handler, formatter: logging.Formatter
+    command_name: str,
+    file_handler: logging.handlers.RotatingFileHandler,
+    formatter: logging.Formatter,
 ) -> None:
     """
     Set up component loggers to also send logs to the command's log file.
@@ -205,7 +213,7 @@ def _setup_component_loggers_for_command(
         command_handler.setFormatter(formatter)
 
         # Add a marker to identify which command this handler belongs to
-        command_handler._command_name = command_name
+        setattr(command_handler, "_command_name", command_name)
 
         # Remove any existing handlers for this command (in case of multiple calls)
         existing_handlers = [
@@ -225,7 +233,7 @@ def _setup_component_loggers_for_command(
 
 
 def log_performance_metrics(
-    logger: logging.Logger, operation: str, duration: float, **metrics
+    logger: logging.Logger, operation: str, duration: float, **metrics: object
 ) -> None:
     """
     Log performance metrics in structured format.

mcp_code_indexer/main.py

@@ -10,10 +10,12 @@ import asyncio
 import json
 import sys
 from pathlib import Path
+from typing import Any, Dict, List, Optional
+import logging
 
 from . import __version__
-from .logging_config import setup_logging
 from .error_handler import setup_error_handling
+from .logging_config import setup_logging
 
 
 def parse_arguments() -> argparse.Namespace:
@@ -77,8 +79,7 @@ def parse_arguments() -> argparse.Namespace:
         nargs="+",
         metavar="PROJECT_ID",
         help=(
-            "Export descriptions for a project. Usage: "
-            "--dumpdescriptions PROJECT_ID"
+            "Export descriptions for a project. Usage: --dumpdescriptions PROJECT_ID"
         ),
     )
 
@@ -105,8 +106,7 @@ def parse_arguments() -> argparse.Namespace:
         type=str,
         metavar="PROJECT_NAME_OR_ID",
         help=(
-            "Generate a markdown project map for the specified project "
-            "(by name or ID)"
+            "Generate a markdown project map for the specified project (by name or ID)"
         ),
     )
 
@@ -122,27 +122,27 @@ def parse_arguments() -> argparse.Namespace:
         action="store_true",
         help="Enable HTTP transport instead of stdio (requires 'http' extras)",
     )
-    
+
     parser.add_argument(
         "--host",
         type=str,
         default="127.0.0.1",
         help="Host to bind HTTP server to (default: 127.0.0.1)",
     )
-    
+
     parser.add_argument(
         "--port",
         type=int,
         default=7557,
         help="Port to bind HTTP server to (default: 7557)",
     )
-    
+
     parser.add_argument(
         "--auth-token",
         type=str,
         help="Bearer token for HTTP authentication (optional)",
     )
-    
+
     parser.add_argument(
         "--cors-origins",
         type=str,
@@ -151,6 +151,19 @@ def parse_arguments() -> argparse.Namespace:
         help="Allowed CORS origins for HTTP transport (default: allow all)",
     )
 
+    # Vector mode options
+    parser.add_argument(
+        "--vector",
+        action="store_true",
+        help="Enable vector mode with semantic search capabilities (requires vector extras)",
+    )
+
+    parser.add_argument(
+        "--vector-config",
+        type=str,
+        help="Path to vector mode configuration file",
+    )
+
     return parser.parse_args()
 
 
@@ -199,8 +212,8 @@ async def handle_getprojects(args: argparse.Namespace) -> None:
 
 async def handle_runcommand(args: argparse.Namespace) -> None:
     """Handle --runcommand command."""
-    from .server.mcp_server import MCPCodeIndexServer
     from .logging_config import setup_command_logger
+    from .server.mcp_server import MCPCodeIndexServer
 
     # Set up dedicated logging for runcommand
     cache_dir = Path(args.cache_dir).expanduser()
@@ -390,8 +403,8 @@ async def handle_runcommand(args: argparse.Namespace) -> None:
 
             return html.unescape(text)
 
-        def clean_arguments(arguments: dict) -> dict:
-            cleaned = {}
+        def clean_arguments(arguments: Dict[str, Any]) -> Dict[str, Any]:
+            cleaned: Dict[str, Any] = {}
             for key, value in arguments.items():
                 if isinstance(value, str):
                     cleaned[key] = clean_html_entities(value)
@@ -753,9 +766,7 @@ async def handle_map(args: argparse.Namespace) -> None:
         )
 
         # Generate markdown
-        markdown_content = generate_project_markdown(
-            project, overview, files, logger
-        )
+        markdown_content = generate_project_markdown(project, overview, files, logger)
 
         # Output the markdown
         print(markdown_content)
@@ -788,7 +799,9 @@ async def handle_map(args: argparse.Namespace) -> None:
             logger.removeHandler(handler)
 
 
-def generate_project_markdown(project, overview, files, logger):
+def generate_project_markdown(
+    project: Any, overview: Optional[Any], files: List[Any], logger: logging.Logger
+) -> str:
     """Generate the markdown content for the project map."""
     import re
     from collections import defaultdict
@@ -881,8 +894,8 @@ def generate_project_markdown(project, overview, files, logger):
 async def handle_makelocal(args: argparse.Namespace) -> None:
     """Handle --makelocal command."""
     try:
-        from .database.database_factory import DatabaseFactory
         from .commands.makelocal import MakeLocalCommand
+        from .database.database_factory import DatabaseFactory
 
         # Initialize database factory
         db_path = Path(args.db_path).expanduser()
@@ -910,10 +923,12 @@ async def handle_makelocal(args: argparse.Namespace) -> None:
         # Execute the command
         result = await makelocal_cmd.execute(args.makelocal)
 
-        print(f"Successfully migrated project '{result['project_name']}' to local database")
+        print(
+            f"Successfully migrated project '{result['project_name']}' to local database"
+        )
         print(f"Local database created at: {result['local_database_path']}")
         print(f"Migrated {result['migrated_files']} file descriptions")
-        if result['migrated_overview']:
+        if result["migrated_overview"]:
             print("Migrated project overview")
 
         # Close all database connections
@@ -994,13 +1009,60 @@ async def main() -> None:
     )
 
     try:
+        # Handle vector mode initialization
+        vector_daemon_task = None
+        if args.vector:
+            try:
+                from .vector_mode import is_vector_mode_available, check_api_keys
+                from .vector_mode.config import load_vector_config
+                from .vector_mode.daemon import start_vector_daemon
+                
+                # Check if vector mode is available
+                if not is_vector_mode_available():
+                    logger.error("Vector mode requires additional dependencies. Install with: pip install mcp-code-indexer[vector]")
+                    sys.exit(1)
+                
+                # Check API keys
+                api_keys = check_api_keys()
+                if not all(api_keys.values()):
+                    missing = [k for k, v in api_keys.items() if not v]
+                    logger.error(f"Missing API keys for vector mode: {', '.join(missing)}")
+                    sys.exit(1)
+                
+                # Load vector configuration
+                vector_config_path = Path(args.vector_config).expanduser() if args.vector_config else None
+                vector_config = load_vector_config(vector_config_path)
+                
+                logger.info(
+                    "Vector mode enabled", 
+                    extra={
+                        "structured_data": {
+                            "embedding_model": vector_config.embedding_model,
+                            "batch_size": vector_config.batch_size,
+                            "daemon_enabled": vector_config.daemon_enabled,
+                        }
+                    }
+                )
+                
+                # Start vector daemon in background
+                if vector_config.daemon_enabled:
+                    vector_daemon_task = asyncio.create_task(
+                        start_vector_daemon(vector_config_path, db_path, cache_dir)
+                    )
+                    logger.info("Vector daemon started")
+                
+            except Exception as e:
+                logger.error(f"Failed to initialize vector mode: {e}")
+                sys.exit(1)
+
         # Import and run the MCP server
         from .server.mcp_server import MCPCodeIndexServer
-        
+
         # Create transport based on arguments
         transport = None
         if args.http:
             from .transport.http_transport import HTTPTransport
+
             transport = HTTPTransport(
                 server_instance=None,  # Will be set after server creation
                 host=args.host,
@@ -1025,8 +1087,9 @@ async def main() -> None:
             db_path=db_path,
             cache_dir=cache_dir,
             transport=transport,
+            vector_mode=args.vector,
         )
-        
+
         # Set server instance in transport after server creation
         if transport:
             transport.server = server
@@ -1038,7 +1101,7 @@ async def main() -> None:
         raise
 
 
-def cli_main():
+def cli_main() -> None:
     """Console script entry point."""
     try:
         asyncio.run(main())

mcp_code_indexer/middleware/__init__.py

@@ -5,8 +5,8 @@ This module provides middleware for HTTP transport features like
 logging, authentication, and security.
 """
 
-from .logging import HTTPLoggingMiddleware
 from .auth import HTTPAuthMiddleware
+from .logging import HTTPLoggingMiddleware
 from .security import HTTPSecurityMiddleware
 
 __all__ = ["HTTPLoggingMiddleware", "HTTPAuthMiddleware", "HTTPSecurityMiddleware"]