mcp-code-indexer 2.2.1__py3-none-any.whl → 2.3.0__py3-none-any.whl

mcp_code_indexer/database/database.py

@@ -30,6 +30,7 @@ from mcp_code_indexer.database.exceptions import (
 from mcp_code_indexer.database.connection_health import (
     ConnectionHealthMonitor, DatabaseMetricsCollector
 )
+from mcp_code_indexer.query_preprocessor import preprocess_search_query
 
 logger = logging.getLogger(__name__)
 
@@ -848,7 +849,16 @@ class DatabaseManager:
         query: str,
         max_results: int = 20
     ) -> List[SearchResult]:
-        """Search file descriptions using FTS5."""
+        """Search file descriptions using FTS5 with intelligent query preprocessing."""
+        # Preprocess query for optimal FTS5 search
+        preprocessed_query = preprocess_search_query(query)
+        
+        if not preprocessed_query:
+            logger.debug(f"Empty query after preprocessing: '{query}'")
+            return []
+        
+        logger.debug(f"Search query preprocessing: '{query}' -> '{preprocessed_query}'")
+        
         async with self.get_connection() as db:
             cursor = await db.execute(
                 """
@@ -866,7 +876,7 @@ class DatabaseManager:
                 ORDER BY bm25(file_descriptions_fts)
                 LIMIT ?
                 """,
-                (query, project_id, branch, max_results)
+                (preprocessed_query, project_id, branch, max_results)
             )
             rows = await cursor.fetchall()
             

mcp_code_indexer/query_preprocessor.py

@@ -0,0 +1,181 @@
+"""
+Query preprocessing module for intelligent FTS5 search.
+
+This module provides intelligent query preprocessing for SQLite FTS5 full-text search
+to enable multi-word search with case insensitive matching, whole word enforcement,
+and proper handling of FTS5 operators as literal search terms.
+
+Key features:
+- Multi-word queries: "grpc proto" becomes "grpc" AND "proto" for order-agnostic matching
+- FTS5 operator escaping: "AND OR" becomes '"AND" AND "OR"' to treat operators as literals
+- Whole word matching: prevents partial matches by relying on proper tokenization
+- Case insensitive: leverages FTS5 default behavior
+- Special character handling: preserves special characters in quoted terms
+"""
+
+import re
+import logging
+from typing import List, Set
+
+logger = logging.getLogger(__name__)
+
+
+class QueryPreprocessor:
+    """
+    Preprocesses user queries for optimal FTS5 search performance.
+    
+    Handles multi-word queries, operator escaping, and special character preservation
+    while maintaining BM25 ranking performance.
+    """
+    
+    # FTS5 operators that need to be escaped when used as literal search terms
+    FTS5_OPERATORS: Set[str] = {
+        'AND', 'OR', 'NOT', 'NEAR'
+    }
+    
+    def __init__(self):
+        """Initialize the query preprocessor."""
+        pass
+    
+    def preprocess_query(self, query: str) -> str:
+        """
+        Preprocess a user query for FTS5 search.
+        
+        Args:
+            query: Raw user query string
+            
+        Returns:
+            Preprocessed query string optimized for FTS5
+            
+        Examples:
+            >>> preprocessor = QueryPreprocessor()
+            >>> preprocessor.preprocess_query("grpc proto")
+            '"grpc" AND "proto"'
+            >>> preprocessor.preprocess_query("error AND handling")
+            '"error" AND "AND" AND "handling"'
+            >>> preprocessor.preprocess_query('config "file system"')
+            '"config" AND "file system"'
+        """
+        if not query or not query.strip():
+            return ""
+        
+        # Normalize whitespace
+        query = query.strip()
+        
+        # Split into terms while preserving quoted phrases
+        terms = self._split_terms(query)
+        
+        if not terms:
+            return ""
+        
+        # Process each term: escape operators and add quotes
+        processed_terms = []
+        for term in terms:
+            processed_term = self._process_term(term)
+            if processed_term:  # Skip empty terms
+                processed_terms.append(processed_term)
+        
+        if not processed_terms:
+            return ""
+        
+        # Join with AND for multi-word matching
+        result = " AND ".join(processed_terms)
+        
+        logger.debug(f"Preprocessed query: '{query}' -> '{result}'")
+        return result
+    
+    def _split_terms(self, query: str) -> List[str]:
+        """
+        Split query into terms while preserving quoted phrases.
+        
+        Args:
+            query: Input query string
+            
+        Returns:
+            List of terms and quoted phrases
+            
+        Examples:
+            'grpc proto' -> ['grpc', 'proto']
+            'config "file system"' -> ['config', '"file system"']
+            'error AND handling' -> ['error', 'AND', 'handling']
+        """
+        terms = []
+        
+        # Regex to match quoted phrases or individual words
+        # This pattern captures:
+        # 1. Double-quoted strings (including the quotes)
+        # 2. Single words (sequences of non-whitespace characters)
+        pattern = r'"[^"]*"|\S+'
+        
+        matches = re.findall(pattern, query)
+        
+        for match in matches:
+            # Skip empty matches
+            if match.strip():
+                terms.append(match)
+        
+        return terms
+    
+    def _process_term(self, term: str) -> str:
+        """
+        Process a single term: escape operators and ensure proper quoting.
+        
+        Args:
+            term: Single term or quoted phrase
+            
+        Returns:
+            Processed term ready for FTS5
+            
+        Examples:
+            'grpc' -> '"grpc"'
+            'AND' -> '"AND"'
+            '"file system"' -> '"file system"'
+            'c++' -> '"c++"'
+        """
+        if not term:
+            return ""
+        
+        # If already quoted, return as-is (user intentional phrase)
+        if term.startswith('"') and term.endswith('"') and len(term) >= 2:
+            return term
+        
+        # Check if term is an FTS5 operator (case-insensitive)
+        if term.upper() in self.FTS5_OPERATORS:
+            # Escape operator by quoting
+            escaped_term = f'"{term}"'
+            logger.debug(f"Escaped FTS5 operator: '{term}' -> '{escaped_term}'")
+            return escaped_term
+        
+        # Quote all terms to ensure whole-word matching and handle special characters
+        return f'"{term}"'
+    
+    def _escape_quotes_in_term(self, term: str) -> str:
+        """
+        Escape internal quotes in a term for FTS5 compatibility.
+        
+        Args:
+            term: Term that may contain quotes
+            
+        Returns:
+            Term with escaped quotes
+            
+        Examples:
+            'say "hello"' -> 'say ""hello""'
+            "test's file" -> "test's file"
+        """
+        # In FTS5, quotes inside quoted strings are escaped by doubling them
+        return term.replace('"', '""')
+
+
+def preprocess_search_query(query: str) -> str:
+    """
+    Convenience function for preprocessing search queries.
+    
+    Args:
+        query: Raw user query
+        
+    Returns:
+        Preprocessed query ready for FTS5
+    """
+    preprocessor = QueryPreprocessor()
+    return preprocessor.preprocess_query(query)

{mcp_code_indexer-2.2.1.dist-info → mcp_code_indexer-2.3.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mcp-code-indexer
-Version: 2.2.1
+Version: 2.3.0
 Summary: MCP server that tracks file descriptions across codebases, enabling AI agents to efficiently navigate and understand code through searchable summaries and token-aware overviews.
 Author: MCP Code Indexer Contributors
 Maintainer: MCP Code Indexer Contributors
@@ -59,8 +59,8 @@ Dynamic: requires-python
 
 # MCP Code Indexer 🚀
 
-[![PyPI version](https://badge.fury.io/py/mcp-code-indexer.svg?16)](https://badge.fury.io/py/mcp-code-indexer)
-[![Python](https://img.shields.io/pypi/pyversions/mcp-code-indexer.svg?16)](https://pypi.org/project/mcp-code-indexer/)
+[![PyPI version](https://badge.fury.io/py/mcp-code-indexer.svg?17)](https://badge.fury.io/py/mcp-code-indexer)
+[![Python](https://img.shields.io/pypi/pyversions/mcp-code-indexer.svg?17)](https://pypi.org/project/mcp-code-indexer/)
 [![License](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT)
 
 A production-ready **Model Context Protocol (MCP) server** that revolutionizes how AI agents navigate and understand codebases. Built for high-concurrency environments with advanced database resilience, the server provides instant access to intelligent descriptions, semantic search, and context-aware recommendations while maintaining 800+ writes/sec throughput.

{mcp_code_indexer-2.2.1.dist-info → mcp_code_indexer-2.3.0.dist-info}/RECORD

@@ -9,11 +9,12 @@ mcp_code_indexer/git_hook_handler.py,sha256=k6QpoLI-5D9EvrLQrHWMII2qNu21daRX_jXl
 mcp_code_indexer/logging_config.py,sha256=tf_U-Zz_axDXRV9s7TfHEeUrBjT1QBWkzPuiyZMffBU,10252
 mcp_code_indexer/main.py,sha256=U-f3AJYdycWhjh-vLryj7aH8DGCs4d3x1yjA852HTxM,31546
 mcp_code_indexer/merge_handler.py,sha256=lJR8eVq2qSrF6MW9mR3Fy8UzrNAaQ7RsI2FMNXne3vQ,14692
+mcp_code_indexer/query_preprocessor.py,sha256=uHYy8FO4FTs7MFKsXoueYIafWDKOIirRgdUzwh8upb4,5773
 mcp_code_indexer/token_counter.py,sha256=WrifOkbF99nWWHlRlhCHAB2KN7qr83GOHl7apE-hJcE,8460
 mcp_code_indexer/data/stop_words_english.txt,sha256=7Zdd9ameVgA6tN_zuXROvHXD4hkWeELVywPhb7FJEkw,6343
 mcp_code_indexer/database/__init__.py,sha256=aPq_aaRp0aSwOBIq9GkuMNjmLxA411zg2vhdrAuHm-w,38
 mcp_code_indexer/database/connection_health.py,sha256=s2r9L_KipH5NlemAUDnhBQO90Dn4b_0Ht9UDs7F6QPk,24432
-mcp_code_indexer/database/database.py,sha256=86XL1b49cTeTzkJ1mVbkYPq_QyQrVQOy8w_b1MxZR-E,50856
+mcp_code_indexer/database/database.py,sha256=7B1Pq9CFwIgU0k8ObcgtGxqdnGPZnDhaYuTTdjf7AV0,51334
 mcp_code_indexer/database/exceptions.py,sha256=AgpRA9Z5R-GoWYdQSPeSdYvAXDopFCQkLGN3jD7Ha4E,10215
 mcp_code_indexer/database/models.py,sha256=_vCmJnPXZSiInRzyvs4c7QUWuNNW8qsOoDlGX8J-Gnk,7124
 mcp_code_indexer/database/retry_executor.py,sha256=QUayjkCk8OsckVMYiJ_HBQ9NTUss-H8GQeUIUbbw4_U,13419
@@ -23,9 +24,9 @@ mcp_code_indexer/server/__init__.py,sha256=16xMcuriUOBlawRqWNBk6niwrvtv_JD5xvI36
 mcp_code_indexer/server/mcp_server.py,sha256=Bu0H8HNqZ6S8_BUZndQLIQpq4cDys3Ry9eEKBuIJItQ,70505
 mcp_code_indexer/tiktoken_cache/9b5ad71b2ce5302211f9c61530b329a4922fc6a4,sha256=Ijkht27pm96ZW3_3OFE-7xAPtR0YyTWXoRO8_-hlsqc,1681126
 mcp_code_indexer/tools/__init__.py,sha256=m01mxML2UdD7y5rih_XNhNSCMzQTz7WQ_T1TeOcYlnE,49
-mcp_code_indexer-2.2.1.dist-info/licenses/LICENSE,sha256=JN9dyPPgYwH9C-UjYM7FLNZjQ6BF7kAzpF3_4PwY4rY,1086
-mcp_code_indexer-2.2.1.dist-info/METADATA,sha256=0EGjVvrBPbpQ-webmSYeXEyxWA0RRORrRGx24-DPloQ,20165
-mcp_code_indexer-2.2.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-mcp_code_indexer-2.2.1.dist-info/entry_points.txt,sha256=8HqWOw1Is7jOP1bvIgaSwouvT9z_Boe-9hd4NzyJOhY,68
-mcp_code_indexer-2.2.1.dist-info/top_level.txt,sha256=yKYCM-gMGt-cnupGfAhnZaoEsROLB6DQ1KFUuyKx4rw,17
-mcp_code_indexer-2.2.1.dist-info/RECORD,,
+mcp_code_indexer-2.3.0.dist-info/licenses/LICENSE,sha256=JN9dyPPgYwH9C-UjYM7FLNZjQ6BF7kAzpF3_4PwY4rY,1086
+mcp_code_indexer-2.3.0.dist-info/METADATA,sha256=lvPoSUWnbJ20uv59kQYBjiCpj9Ea6l439haNAK4WDQU,20165
+mcp_code_indexer-2.3.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+mcp_code_indexer-2.3.0.dist-info/entry_points.txt,sha256=8HqWOw1Is7jOP1bvIgaSwouvT9z_Boe-9hd4NzyJOhY,68
+mcp_code_indexer-2.3.0.dist-info/top_level.txt,sha256=yKYCM-gMGt-cnupGfAhnZaoEsROLB6DQ1KFUuyKx4rw,17
+mcp_code_indexer-2.3.0.dist-info/RECORD,,