code-finder 0.1.0__py3-none-any.whl

claude_context/indexer.py

@@ -0,0 +1,893 @@
+"""
+Repository Indexer for Claude Context
+
+This module handles discovering, chunking, and indexing code repositories
+for semantic search. Supports multiple repository sources through adapters.
+Follows fail-fast principles with clear logging.
+"""
+
+import hashlib
+import logging
+from pathlib import Path
+from typing import List, Dict, Any, Optional, Tuple, Union, Iterable
+from datetime import datetime
+import json
+import tempfile
+import time
+
+import numpy as np
+from tqdm import tqdm
+from pymilvus import Collection, FieldSchema, CollectionSchema, DataType, MilvusClient
+
+from .config import ClaudeContextConfig, MilvusManager
+from .embeddings import LocalEmbeddings
+from .ast_chunker import ASTChunker, CodeChunk as ASTCodeChunk
+
+# Import enhanced chunker with fallback to basic chunker
+try:
+    from .enhanced_ast_chunker import EnhancedASTChunker, EnhancedCodeChunk, HAS_TREE_SITTER
+    HAS_ENHANCED_CHUNKER = HAS_TREE_SITTER
+except ImportError:
+    HAS_ENHANCED_CHUNKER = False
+
+# Import markdown chunker for README and documentation files
+try:
+    from .markdown_chunker import MarkdownChunker, MarkdownChunk
+    HAS_MARKDOWN_CHUNKER = True
+except ImportError:
+    HAS_MARKDOWN_CHUNKER = False
+
+logger = logging.getLogger(__name__)
+
+
+class RepositoryIndexer:
+    """
+    Index repository files for semantic search.
+    
+    This class handles the complete indexing pipeline:
+    1. Discover files based on extensions and ignore patterns
+    2. Chunk files into semantic units
+    3. Generate embeddings for each chunk
+    4. Store in Milvus for vector search
+    """
+    
+    def __init__(
+        self,
+        config: ClaudeContextConfig,
+        embeddings: LocalEmbeddings,
+        milvus_manager: MilvusManager
+    ):
+        """
+        Initialize the indexer.
+        
+        Args:
+            config: Configuration with file patterns and chunking settings
+            embeddings: Embeddings model for vectorization
+            milvus_manager: Milvus connection manager
+            
+        Raises:
+            ValueError: If configuration is invalid
+        """
+        if not config:
+            raise ValueError("config is required")
+        if not embeddings:
+            raise ValueError("embeddings is required")
+        if not milvus_manager:
+            raise ValueError("milvus_manager is required")
+        
+        self.config = config
+        self.embeddings = embeddings
+        self.milvus_client = milvus_manager.get_client()
+        
+        # Initialize chunker - prefer enhanced chunker for richer metadata
+        self.use_enhanced_chunker = HAS_ENHANCED_CHUNKER
+
+        if self.use_enhanced_chunker:
+            self.enhanced_chunker = EnhancedASTChunker(
+                max_chunk_size=config.chunk_size,
+                context_mode="full"  # Full context for best retrieval
+            )
+            # Keep basic chunker as fallback
+            self.ast_chunker = ASTChunker(
+                max_chunk_size=config.chunk_size // 100,
+                chunk_overlap=config.chunk_overlap // 100
+            )
+            logger.info("RepositoryIndexer initialized with ENHANCED AST chunking (+44% metadata richness)")
+        else:
+            self.ast_chunker = ASTChunker(
+                max_chunk_size=config.chunk_size // 100,
+                chunk_overlap=config.chunk_overlap // 100
+            )
+            logger.info("RepositoryIndexer initialized with basic AST chunking")
+
+        # Initialize markdown chunker for README and documentation files
+        self.use_markdown_chunker = HAS_MARKDOWN_CHUNKER
+        if self.use_markdown_chunker:
+            self.markdown_chunker = MarkdownChunker(
+                max_chunk_size=config.chunk_size,
+                extract_code_blocks=True,
+                include_header_context=True
+            )
+            logger.info("MarkdownChunker enabled for README/documentation files")
+
+        # Collection name for this repository
+        self.collection_name = "code_chunks"
+
+        # Track indexing statistics
+        self.stats = {
+            "files_discovered": 0,
+            "files_indexed": 0,
+            "chunks_created": 0,
+            "chunks_by_type": {},  # Track chunk types
+            "errors": [],
+            "chunker_type": "enhanced" if self.use_enhanced_chunker else "basic"
+        }
+
+        # Cache of most recently indexed chunks for downstream summarization (trimmed for safety)
+        self.last_indexed_chunks: List[Dict[str, Any]] = []
+    
+    def discover_files(self, repo_path: str) -> List[Path]:
+        """
+        Discover files to index in the repository.
+        
+        Args:
+            repo_path: Path to the repository
+            
+        Returns:
+            List of file paths to index
+            
+        Raises:
+            ValueError: If repo_path doesn't exist or isn't a directory
+        """
+        repo = Path(repo_path)
+        
+        # Validate path - fail fast
+        if not repo.exists():
+            raise ValueError(f"Repository path does not exist: {repo_path}")
+        if not repo.is_dir():
+            raise ValueError(f"Repository path is not a directory: {repo_path}")
+        
+        logger.info(f"Discovering files in: {repo.absolute()}")
+        logger.info(f"Looking for extensions: {self.config.file_extensions[:5]}...")
+        
+        discovered_files = []
+        
+        # Discover files by extension
+        for ext in self.config.file_extensions:
+            pattern = f"**/*{ext}"
+            files = list(repo.glob(pattern))
+            logger.debug(f"Found {len(files)} files with extension {ext}")
+            discovered_files.extend(files)
+        
+        # Remove duplicates
+        discovered_files = list(set(discovered_files))
+        
+        # Apply ignore patterns
+        filtered_files = []
+        for file_path in discovered_files:
+            relative_path = str(file_path.relative_to(repo))
+            
+            # Check against ignore patterns
+            should_ignore = False
+            for pattern in self.config.ignore_patterns:
+                if self._matches_pattern(relative_path, pattern):
+                    logger.debug(f"Ignoring {relative_path} (matches {pattern})")
+                    should_ignore = True
+                    break
+            
+            if not should_ignore:
+                filtered_files.append(file_path)
+        
+        self.stats["files_discovered"] = len(filtered_files)
+        logger.info(f"Discovered {len(filtered_files)} files to index")
+        
+        return sorted(filtered_files)  # Sort for consistent ordering
+    
+    def chunk_file(self, file_path: Path) -> List[Dict[str, Any]]:
+        """
+        Chunk a file into semantic units using appropriate chunker.
+
+        Uses different chunkers based on file type:
+        - Markdown files (.md): MarkdownChunker for section-aware chunking
+        - Code files: EnhancedASTChunker or basic ASTChunker
+
+        Args:
+            file_path: Path to the file to chunk
+
+        Returns:
+            List of chunks with metadata
+        """
+        chunks = []
+
+        try:
+            # Use markdown chunker for .md files
+            if self.use_markdown_chunker and file_path.suffix.lower() == '.md':
+                return self._chunk_markdown_file(file_path)
+
+            if self.use_enhanced_chunker:
+                # Use enhanced chunker for richer metadata
+                enhanced_chunks = self.enhanced_chunker.chunk_file(file_path)
+
+                if not enhanced_chunks:
+                    logger.debug(f"No chunks created from {file_path}")
+                    return []
+
+                for chunk in enhanced_chunks:
+                    # Build scope string from scope chain
+                    scope_str = " > ".join(chunk.scope) if chunk.scope else ""
+
+                    # Build signature info
+                    signature = chunk.signature or ""
+
+                    # Build imports list
+                    imports_list = [imp.name for imp in chunk.imports] if chunk.imports else []
+
+                    chunk_dict = {
+                        "id": self._generate_chunk_id(file_path, chunk.line_range[0], chunk.text),
+                        "content": chunk.text,
+                        "contextualized_content": chunk.contextualized_text,  # For embeddings
+                        "file_path": str(file_path),
+                        "file_name": file_path.name,
+                        "start_line": chunk.line_range[0] + 1,  # Convert to 1-indexed
+                        "end_line": chunk.line_range[1] + 1,
+                        "language": chunk.language,
+                        "chunk_size": chunk.size_chars,
+                        "chunk_type": chunk.chunk_type,
+                        "chunk_name": chunk.name or "",
+                        "parent_context": scope_str,  # Full scope chain
+                        "signature": signature,
+                        "docstring": chunk.docstring or "",
+                        "imports": imports_list,
+                        "return_type": chunk.return_type or "",
+                        "indexed_at": datetime.now().isoformat()
+                    }
+                    chunks.append(chunk_dict)
+
+                    # Track chunk types in stats
+                    chunk_type = chunk.chunk_type
+                    if chunk_type not in self.stats["chunks_by_type"]:
+                        self.stats["chunks_by_type"][chunk_type] = 0
+                    self.stats["chunks_by_type"][chunk_type] += 1
+
+                logger.debug(f"Created {len(chunks)} enhanced chunks from {file_path.name}")
+
+            else:
+                # Fallback to basic AST chunker
+                ast_chunks = self.ast_chunker.chunk_file(file_path)
+
+                if not ast_chunks:
+                    logger.debug(f"No chunks created from {file_path}")
+                    return []
+
+                for ast_chunk in ast_chunks:
+                    chunk_dict = {
+                        "id": self._generate_chunk_id(file_path, ast_chunk.start_line, ast_chunk.content),
+                        "content": ast_chunk.content,
+                        "contextualized_content": ast_chunk.content,  # Same as content for basic
+                        "file_path": str(file_path),
+                        "file_name": file_path.name,
+                        "start_line": ast_chunk.start_line,
+                        "end_line": ast_chunk.end_line,
+                        "language": ast_chunk.language,
+                        "chunk_size": ast_chunk.size_chars,
+                        "chunk_type": ast_chunk.chunk_type,
+                        "chunk_name": ast_chunk.name or "",
+                        "parent_context": ast_chunk.parent_context or "",
+                        "signature": "",
+                        "docstring": "",
+                        "imports": [],
+                        "return_type": "",
+                        "indexed_at": datetime.now().isoformat()
+                    }
+                    chunks.append(chunk_dict)
+
+                    chunk_type = ast_chunk.chunk_type
+                    if chunk_type not in self.stats["chunks_by_type"]:
+                        self.stats["chunks_by_type"][chunk_type] = 0
+                    self.stats["chunks_by_type"][chunk_type] += 1
+
+                logger.debug(f"Created {len(chunks)} basic chunks from {file_path.name}")
+
+        except Exception as e:
+            logger.error(f"Error chunking file {file_path}: {e}")
+            self.stats["errors"].append({"file": str(file_path), "error": str(e)})
+            return []
+
+        return chunks
+
+    def _chunk_markdown_file(self, file_path: Path) -> List[Dict[str, Any]]:
+        """
+        Chunk a markdown file using the MarkdownChunker.
+
+        Extracts:
+        - Sections with header hierarchy as scope chains
+        - Code blocks as separate chunks with language tags
+        - Links and references
+
+        Args:
+            file_path: Path to the markdown file
+
+        Returns:
+            List of chunks with metadata
+        """
+        chunks = []
+
+        try:
+            md_chunks = self.markdown_chunker.chunk_file(file_path)
+
+            if not md_chunks:
+                logger.debug(f"No chunks created from markdown file {file_path}")
+                return []
+
+            for chunk in md_chunks:
+                # Build scope string from header hierarchy
+                scope_str = " > ".join(chunk.scope) if chunk.scope else ""
+
+                # Extract links as JSON for storage
+                links_json = json.dumps(chunk.links) if chunk.links else "[]"
+
+                chunk_dict = {
+                    "id": self._generate_chunk_id(file_path, chunk.line_range[0], chunk.text),
+                    "content": chunk.text,
+                    "contextualized_content": chunk.contextualized_text,  # Includes header context
+                    "file_path": str(file_path),
+                    "file_name": file_path.name,
+                    "start_line": chunk.line_range[0] + 1,  # Convert to 1-indexed
+                    "end_line": chunk.line_range[1] + 1,
+                    "language": chunk.code_language or "markdown",
+                    "chunk_size": chunk.size_chars,
+                    "chunk_type": chunk.chunk_type,  # 'section' or 'code_block'
+                    "chunk_name": chunk.name or "",
+                    "parent_context": scope_str,  # Header hierarchy
+                    "signature": "",  # Not applicable for markdown
+                    "docstring": chunk.text[:500] if chunk.chunk_type == 'section' else "",  # Use section text as docstring
+                    "imports": links_json,  # Repurpose imports field for links
+                    "return_type": "",
+                    "indexed_at": datetime.now().isoformat()
+                }
+                chunks.append(chunk_dict)
+
+                # Track chunk types in stats
+                chunk_type = f"md_{chunk.chunk_type}"
+                if chunk_type not in self.stats["chunks_by_type"]:
+                    self.stats["chunks_by_type"][chunk_type] = 0
+                self.stats["chunks_by_type"][chunk_type] += 1
+
+            logger.debug(f"Created {len(chunks)} markdown chunks from {file_path.name}")
+
+        except Exception as e:
+            logger.error(f"Error chunking markdown file {file_path}: {e}")
+            self.stats["errors"].append({"file": str(file_path), "error": str(e)})
+            return []
+
+        return chunks
+
+    def _create_chunk_dict(
+        self,
+        file_path: Path,
+        content: str,
+        start_line: int,
+        end_line: int
+    ) -> Dict[str, Any]:
+        """Create a chunk dictionary with metadata."""
+        # Generate unique ID for this chunk
+        chunk_id = self._generate_chunk_id(file_path, start_line, content)
+        
+        return {
+            "id": chunk_id,
+            "content": content,
+            "file_path": str(file_path),
+            "file_name": file_path.name,
+            "start_line": start_line,
+            "end_line": end_line,
+            "language": self._detect_language(file_path),
+            "chunk_size": len(content),
+            "indexed_at": datetime.now().isoformat()
+        }
+    
+    def _generate_chunk_id(self, file_path: Path, start_line: int, content: str) -> str:
+        """Generate a unique ID for a chunk."""
+        # Use file path, line number, and content hash for uniqueness
+        id_string = f"{file_path}:{start_line}:{hashlib.md5(content.encode()).hexdigest()[:8]}"
+        return hashlib.sha256(id_string.encode()).hexdigest()[:16]
+    
+    def _detect_language(self, file_path: Path) -> str:
+        """Detect programming language from file extension."""
+        ext_to_lang = {
+            ".py": "python",
+            ".js": "javascript",
+            ".ts": "typescript",
+            ".jsx": "javascript",
+            ".tsx": "typescript",
+            ".java": "java",
+            ".cpp": "cpp",
+            ".c": "c",
+            ".h": "c",
+            ".hpp": "cpp",
+            ".go": "go",
+            ".rs": "rust",
+            ".rb": "ruby",
+            ".php": "php",
+            ".cs": "csharp",
+            ".md": "markdown",
+            ".yaml": "yaml",
+            ".yml": "yaml",
+            ".json": "json",
+            ".xml": "xml",
+            ".html": "html",
+            ".css": "css",
+            ".sql": "sql",
+            ".sh": "shell",
+            ".bash": "bash",
+        }
+        return ext_to_lang.get(file_path.suffix.lower(), "text")
+    
+    def _matches_pattern(self, path: str, pattern: str) -> bool:
+        """Check if a path matches an ignore pattern."""
+        import fnmatch
+        
+        # Handle ** wildcards
+        if "**" in pattern:
+            # Convert ** to Python's glob pattern
+            pattern = pattern.replace("**", "*")
+        
+        return fnmatch.fnmatch(path, pattern) or fnmatch.fnmatch(f"/{path}", pattern)
+    
+    def _ensure_collection_exists(self) -> None:
+        """Ensure the Milvus collection exists without dropping existing data."""
+        if not self.milvus_client.has_collection(self.collection_name):
+            self.create_collection()
+
+    def create_collection(self) -> None:
+        """
+        Create or recreate the Milvus collection for storing chunks.
+        
+        Raises:
+            RuntimeError: If collection creation fails
+        """
+        logger.info(f"Setting up collection: {self.collection_name}")
+        
+        # Check if collection exists
+        if self.milvus_client.has_collection(self.collection_name):
+            logger.info(f"Dropping existing collection: {self.collection_name}")
+            self.milvus_client.drop_collection(self.collection_name)
+        
+        # Create new collection with schema
+        dimension = self.embeddings.get_dimension()
+        
+        # Create collection with auto-id and dynamic schema
+        self.milvus_client.create_collection(
+            collection_name=self.collection_name,
+            dimension=dimension,
+            metric_type="COSINE",  # Cosine similarity for semantic search
+            auto_id=True  # Let Milvus generate IDs
+        )
+        
+        logger.info(f"✅ Collection created: {self.collection_name} (dimension={dimension})")
+    
+    def index_repository(
+        self,
+        repo_path: str,
+        show_progress: bool = True
+    ) -> Dict[str, Any]:
+        """
+        Index an entire repository.
+        
+        Args:
+            repo_path: Path to the repository
+            show_progress: Whether to show progress bars
+            
+        Returns:
+            Dictionary with indexing statistics
+            
+        Raises:
+            ValueError: If repo_path is invalid
+            RuntimeError: If indexing fails
+        """
+        start_time = datetime.now()
+        logger.info(f"Starting repository indexing: {repo_path}")
+        
+        # Create/recreate collection
+        self.create_collection()
+        
+        # Discover files
+        files = self.discover_files(repo_path)
+        if not files:
+            logger.warning("No files found to index")
+            return self.stats
+        
+        # Process files
+        all_chunks = []
+        file_iterator = tqdm(files, desc="Processing files") if show_progress else files
+        
+        for file_path in file_iterator:
+            chunks = self.chunk_file(file_path)
+            if chunks:
+                all_chunks.extend(chunks)
+                self.stats["files_indexed"] += 1
+        
+        if not all_chunks:
+            logger.warning("No chunks created from files")
+            return self.stats
+        
+        # Cache a limited view of indexed chunks for downstream summarization/orchestration
+        self.last_indexed_chunks = all_chunks[:200]
+
+        self.stats["chunks_created"] = len(all_chunks)
+        logger.info(f"Created {len(all_chunks)} chunks from {self.stats['files_indexed']} files")
+
+        # Generate embeddings using contextualized content for better semantic matching
+        # contextualized_content includes scope chain, signature, imports - improves retrieval
+        logger.info("Generating embeddings (using contextualized content)...")
+        chunk_texts = [chunk.get("contextualized_content", chunk["content"]) for chunk in all_chunks]
+
+        # Process in batches for memory efficiency
+        batch_size = 100
+        embeddings_list = []
+
+        batch_iterator = range(0, len(chunk_texts), batch_size)
+        if show_progress:
+            batch_iterator = tqdm(batch_iterator, desc="Generating embeddings")
+
+        for i in batch_iterator:
+            batch = chunk_texts[i:i + batch_size]
+            batch_embeddings = self.embeddings.embed_texts(batch, batch_size=32)
+            embeddings_list.append(batch_embeddings)
+        
+        # Combine all embeddings
+        all_embeddings = np.vstack(embeddings_list)
+        logger.info(f"Generated embeddings: shape={all_embeddings.shape}")
+        
+        # Prepare data for insertion
+        logger.info("Inserting into Milvus...")
+
+        # Prepare data in format Milvus expects
+        # Includes new enhanced fields: signature, docstring, imports, return_type
+        data = []
+        for i, chunk in enumerate(all_chunks):
+            # Convert imports list to JSON string for storage
+            imports_json = json.dumps(chunk.get("imports", []))
+
+            data.append({
+                # Don't include "id" - let Milvus auto-generate it
+                "vector": all_embeddings[i].tolist(),
+                "chunk_id": chunk["id"],  # Store our ID as metadata
+                "content": chunk["content"],
+                "file_path": chunk["file_path"],
+                "file_name": chunk["file_name"],
+                "start_line": chunk["start_line"],
+                "end_line": chunk["end_line"],
+                "language": chunk["language"],
+                "chunk_size": chunk["chunk_size"],
+                "chunk_type": chunk.get("chunk_type", "unknown"),
+                "chunk_name": chunk.get("chunk_name", ""),
+                "parent_context": chunk.get("parent_context", ""),
+                # New enhanced fields
+                "signature": chunk.get("signature", ""),
+                "docstring": chunk.get("docstring", "")[:500] if chunk.get("docstring") else "",  # Truncate long docstrings
+                "imports": imports_json,
+                "return_type": chunk.get("return_type", "")
+            })
+        
+        # Insert in batches
+        insert_batch_size = 100
+        for i in range(0, len(data), insert_batch_size):
+            batch = data[i:i + insert_batch_size]
+            self.milvus_client.insert(
+                collection_name=self.collection_name,
+                data=batch
+            )
+        
+        logger.info(f"✅ Inserted {len(data)} chunks into Milvus")
+        
+        # Calculate indexing time
+        elapsed = (datetime.now() - start_time).total_seconds()
+        self.stats["indexing_time_seconds"] = elapsed
+        
+        logger.info(f"Indexing complete in {elapsed:.2f} seconds")
+        logger.info(f"Stats: Files: {self.stats['files_indexed']}/{self.stats['files_discovered']}, "
+                   f"Chunks: {self.stats['chunks_created']}")
+        if self.stats["chunks_by_type"]:
+            logger.info(f"Chunk types: {self.stats['chunks_by_type']}")
+        
+        return self.stats
+    
+    def index_from_adapter(
+        self,
+        adapter: Any,  # RepositoryAdapter type
+        show_progress: bool = True
+    ) -> Dict[str, Any]:
+        """
+        Index a repository using any adapter (flexible source support).
+        
+        This method supports indexing from various sources:
+        - Local filesystem (LocalRepositoryAdapter)
+        - Git repositories (GitCloneAdapter)
+        - GitHub API (GitHubAPIAdapter)
+        - In-memory content (MemoryRepositoryAdapter)
+        
+        Args:
+            adapter: Repository adapter providing file access
+            show_progress: Whether to show progress bars
+            
+        Returns:
+            Dictionary with indexing statistics
+        """
+        start_time = time.time()
+        adapter_info = adapter.get_info()
+        logger.info(f"Starting indexing from {adapter_info['type']} adapter")
+        
+        # Create/recreate collection
+        self.create_collection()
+        
+        # Reset stats
+        self.stats = {
+            "repository_type": adapter_info["type"],
+            "repository_info": adapter_info,
+            "files_indexed": 0,
+            "chunks_created": 0,
+            "chunks_by_type": {},
+            "indexing_time": 0.0
+        }
+        
+        # Collect chunks from adapter
+        all_chunks = []
+        files_processed = 0
+        
+        # Get files from adapter
+        files_iterator = adapter.get_files(self.config.file_extensions)
+        files_list = list(files_iterator) if show_progress else files_iterator
+        
+        if show_progress:
+            files_list = tqdm(files_list, desc="Processing files")
+        
+        for file in files_list:
+            try:
+                # Create temporary file for AST chunker
+                # TODO: Refactor AST chunker to accept content strings directly
+                with tempfile.NamedTemporaryFile(
+                    mode='w',
+                    suffix=Path(file.path).suffix,
+                    delete=False,
+                    encoding='utf-8'
+                ) as tmp:
+                    tmp.write(file.content)
+                    tmp_path = Path(tmp.name)
+                
+                try:
+                    # Use AST chunker
+                    chunks = self.chunk_file(tmp_path)
+                    
+                    # Update chunk metadata with original path
+                    for chunk in chunks:
+                        chunk["file_path"] = file.path
+                        chunk["file_name"] = file.name
+                        
+                        # Track chunk types
+                        chunk_type = chunk.get("chunk_type", "unknown")
+                        self.stats["chunks_by_type"][chunk_type] = \
+                            self.stats["chunks_by_type"].get(chunk_type, 0) + 1
+                    
+                    all_chunks.extend(chunks)
+                    files_processed += 1
+                    
+                finally:
+                    # Clean up temp file
+                    tmp_path.unlink(missing_ok=True)
+                    
+            except Exception as e:
+                logger.warning(f"Failed to process {file.path}: {e}")
+        
+        self.stats["files_indexed"] = files_processed
+        self.stats["chunks_created"] = len(all_chunks)
+        
+        logger.info(f"Created {len(all_chunks)} chunks from {files_processed} files")
+        
+        if not all_chunks:
+            logger.warning("No chunks created from files")
+            return self.stats
+        
+        self.last_indexed_chunks = all_chunks[:200]
+
+        # Generate embeddings using contextualized content for better semantic matching
+        logger.info("Generating embeddings (using contextualized content)...")
+        chunk_texts = [chunk.get("contextualized_content", chunk["content"]) for chunk in all_chunks]
+
+        # Process in batches
+        batch_size = 100
+        embeddings_list = []
+
+        for i in range(0, len(chunk_texts), batch_size):
+            batch = chunk_texts[i:i+batch_size]
+            batch_embeddings = self.embeddings.embed_texts(batch, batch_size=32)
+            embeddings_list.append(batch_embeddings)
+
+        all_embeddings = np.vstack(embeddings_list) if embeddings_list else np.array([])
+        logger.info(f"Generated embeddings: shape={all_embeddings.shape}")
+
+        # Prepare data for Milvus with enhanced fields
+        data = []
+        for i, chunk in enumerate(all_chunks):
+            imports_json = json.dumps(chunk.get("imports", []))
+
+            data.append({
+                "vector": all_embeddings[i].tolist(),
+                "content": chunk["content"],
+                "file_path": chunk["file_path"],
+                "file_name": chunk["file_name"],
+                "start_line": chunk["start_line"],
+                "end_line": chunk["end_line"],
+                "language": chunk["language"],
+                "chunk_type": chunk.get("chunk_type", "unknown"),
+                "chunk_name": chunk.get("chunk_name", ""),
+                "parent_context": chunk.get("parent_context", ""),
+                "chunk_id": chunk["id"],
+                # New enhanced fields
+                "signature": chunk.get("signature", ""),
+                "docstring": chunk.get("docstring", "")[:500] if chunk.get("docstring") else "",
+                "imports": imports_json,
+                "return_type": chunk.get("return_type", "")
+            })
+        
+        # Insert into Milvus
+        logger.info("Inserting into Milvus...")
+        self.milvus_client.insert(
+            collection_name=self.collection_name,
+            data=data
+        )
+        logger.info(f"✅ Inserted {len(data)} chunks into Milvus")
+        
+        # Calculate final stats
+        end_time = time.time()
+        self.stats["indexing_time"] = end_time - start_time
+        
+        logger.info(f"Indexing complete in {self.stats['indexing_time']:.2f} seconds")
+        logger.info(f"Stats: Files: {files_processed}, Chunks: {len(all_chunks)}")
+        logger.info(f"Chunk types: {self.stats['chunks_by_type']}")
+        
+        return self.stats
+    
+    def index_any_repository(
+        self,
+        source: str,
+        show_progress: bool = True,
+        **kwargs
+    ) -> Dict[str, Any]:
+        """
+        Index a repository from any supported source.
+        
+        Automatically detects the source type and uses the appropriate adapter.
+        
+        Args:
+            source: Repository source (path, URL, or special format)
+            show_progress: Whether to show progress bars
+            **kwargs: Additional arguments for the adapter
+            
+        Examples:
+            # Local repository
+            indexer.index_any_repository("/path/to/repo")
+            
+            # Git repository
+            indexer.index_any_repository("https://github.com/user/repo.git")
+            
+            # GitHub API
+            indexer.index_any_repository("github:facebook/react", branch="main")
+            
+        Returns:
+            Indexing statistics
+        """
+        from .repository_adapter import create_repository_adapter, GitCloneAdapter
+        
+        adapter = create_repository_adapter(source, **kwargs)
+        
+        # Handle adapters that need context management
+        if isinstance(adapter, GitCloneAdapter):
+            with adapter:
+                return self.index_from_adapter(adapter, show_progress)
+        else:
+            return self.index_from_adapter(adapter, show_progress)
+
+    def upsert_rationale_entries(self, entries: Iterable[Dict[str, Any]]) -> None:
+        """Insert or update rationale records so they can be retrieved alongside code chunks."""
+        prepared: List[Dict[str, Any]] = []
+        for entry in entries or []:
+            if not entry:
+                continue
+            text = (entry.get("text") or "").strip()
+            if not text:
+                continue
+            prepared.append({**entry, "text": text})
+
+        if not prepared:
+            return
+
+        self._ensure_collection_exists()
+
+        texts = [p["text"] for p in prepared]
+        batch_size = min(32, len(texts)) or 1
+        try:
+            vectors = self.embeddings.embed_texts(texts, batch_size=batch_size)
+        except Exception as exc:  # pragma: no cover - defensive
+            logger.error("Failed to embed rationale entries: %s", exc)
+            return
+
+        payloads: List[Dict[str, Any]] = []
+        for entry, vector in zip(prepared, vectors):
+            record_id = entry.get("record_id") or self._generate_rationale_chunk_id(entry)
+
+            # Remove previous copy if present
+            try:
+                self.milvus_client.delete(
+                    collection_name=self.collection_name,
+                    filter=f'chunk_id == "{record_id}"'
+                )
+            except Exception:
+                logger.debug("No existing rationale chunk to delete for %s", record_id)
+
+            payloads.append({
+                "vector": vector.tolist(),
+                "chunk_id": record_id,
+                "content": entry["text"],
+                "file_path": entry.get("source_path", ""),
+                "file_name": entry.get("source_name", entry.get("title", "")),
+                "start_line": entry.get("start_line", 0),
+                "end_line": entry.get("end_line", 0),
+                "language": entry.get("language", "text"),
+                "chunk_size": len(entry["text"]),
+                "chunk_type": entry.get("record_type", "rationale"),
+                "chunk_name": entry.get("title", ""),
+                "parent_context": entry.get("parent_context", ""),
+            })
+
+        if not payloads:
+            return
+
+        self.milvus_client.insert(
+            collection_name=self.collection_name,
+            data=payloads
+        )
+        try:
+            self.milvus_client.flush(self.collection_name)
+        except Exception:
+            logger.debug("Milvus flush not available for rationale entries")
+
+    @staticmethod
+    def _generate_rationale_chunk_id(entry: Dict[str, Any]) -> str:
+        base = entry.get("record_id") or entry.get("text") or "rationale"
+        return hashlib.sha256(base.encode("utf-8")).hexdigest()[:32]
+
+
+# Example usage
+if __name__ == "__main__":
+    print("Testing Repository Indexer")
+    print("-" * 50)
+    
+    # Setup
+    from .config import ClaudeContextConfig, MilvusManager
+    from .embeddings import LocalEmbeddings
+    
+    config = ClaudeContextConfig()
+    embeddings = LocalEmbeddings()
+    
+    milvus_manager = MilvusManager(config)
+    with milvus_manager:
+        indexer = RepositoryIndexer(config, embeddings, milvus_manager)
+        
+        # Test with the claude_context module itself
+        test_repo = "./src/claude_context"
+        
+        print(f"\nIndexing: {test_repo}")
+        stats = indexer.index_repository(test_repo, show_progress=True)
+        
+        print("\n📊 Indexing Statistics:")
+        print(f"  Files discovered: {stats['files_discovered']}")
+        print(f"  Files indexed: {stats['files_indexed']}")
+        print(f"  Chunks created: {stats['chunks_created']}")
+        print(f"  Time taken: {stats.get('indexing_time_seconds', 0):.2f}s")
+        
+        if stats["errors"]:
+            print(f"  Errors: {len(stats['errors'])}")
+            for error in stats["errors"][:3]:
+                print(f"    - {error}")
+        
+        print("\n✅ Indexer test complete!")