contextual-engine 0.1.0__py3-none-any.whl

contextual/indexing/embedder.py

@@ -0,0 +1,271 @@
+"""Embedding provider abstraction using fastembed/Jina.
+
+Provides a simple interface for the indexing pipeline while delegating
+to contextual.embedding.CodeEmbedder for actual embedding generation.
+
+Key features:
+- Singleton pattern for shared embedder instance
+- Batch processing support
+- 768-dimensional embeddings (jina-v2-code)
+- Content-hash caching handled by CodeEmbedder layer
+"""
+
+from __future__ import annotations
+
+from typing import Protocol
+
+from contextual.embedding.embedder import CodeEmbedder
+from contextual.observability.logging import get_logger
+
+
+logger = get_logger(__name__)
+
+
+class EmbeddingProvider(Protocol):
+    """Protocol for embedding providers.
+    
+    Defines interface that all embedding implementations must follow.
+    """
+
+    def embed(self, texts: list[str]) -> list[list[float]]:
+        """Generate embeddings for batch of texts.
+        
+        Args:
+            texts: List of text strings to embed.
+            
+        Returns:
+            List of embedding vectors (same order as input).
+            
+        Raises:
+            Exception: If embedding generation fails.
+        """
+        ...
+
+    def embed_batch(
+        self,
+        texts: list[str],
+        batch_size: int = 64,
+    ) -> list[list[float]]:
+        """Embed large list of texts in batches.
+        
+        Args:
+            texts: List of texts to embed.
+            batch_size: Number of texts per batch.
+            
+        Returns:
+            List of embedding vectors (same order as input).
+        """
+        ...
+
+    @property
+    def dimension(self) -> int:
+        """Embedding dimension for this provider."""
+        ...
+
+    @property
+    def model_name(self) -> str:
+        """Model identifier."""
+        ...
+
+
+class JinaEmbedder:
+    """Jina embedding provider via fastembed.
+    
+    Wraps contextual.embedding.CodeEmbedder to provide a consistent
+    interface for the indexing pipeline.
+    
+    Uses jina-embeddings-v2-base-code model (768-dim) optimized for code.
+    Content-hash caching is handled by the underlying CodeEmbedder.
+    
+    Attributes:
+        dimension: 768 (fixed for jina-v2-code)
+        model_name: "jina-v2-code"
+    """
+
+    def __init__(self):
+        """Initialize Jina embedder.
+        
+        Creates underlying CodeEmbedder instance which handles:
+        - Model loading (fastembed with jina-v2-code)
+        - Content-hash caching (SQLite-backed)
+        - Batch processing optimization
+        """
+        self._embedder = CodeEmbedder()
+        self._dimension = 768
+        self._model_name = "jina-v2-code"
+
+        logger.info(
+            "JinaEmbedder initialized",
+            model=self._model_name,
+            dimension=self._dimension,
+            provider="fastembed",
+        )
+
+    @property
+    def dimension(self) -> int:
+        """Embedding dimension (768 for jina-v2-code)."""
+        return self._dimension
+
+    @property
+    def model_name(self) -> str:
+        """Model identifier."""
+        return self._model_name
+
+    def embed(self, texts: list[str]) -> list[list[float]]:
+        """Generate embeddings for batch of texts.
+        
+        Delegates to CodeEmbedder.embed() which handles:
+        - Content-hash based caching
+        - Batch processing (64 texts per batch)
+        - Model inference via fastembed
+        
+        Args:
+            texts: List of text strings to embed.
+            
+        Returns:
+            List of 768-dim embedding vectors (same order as input).
+            
+        Raises:
+            Exception: If embedding generation fails.
+        """
+        if not texts:
+            return []
+
+        # CodeEmbedder.embed() handles caching and batching internally
+        embeddings = self._embedder.embed(texts)
+
+        # Validate output
+        if len(embeddings) != len(texts):
+            raise ValueError(
+                f"Embedding count mismatch: got {len(embeddings)}, expected {len(texts)}"
+            )
+
+        # Validate dimensions
+        for i, embedding in enumerate(embeddings):
+            if len(embedding) != self._dimension:
+                raise ValueError(
+                    f"Invalid embedding dimension at index {i}: "
+                    f"got {len(embedding)}, expected {self._dimension}"
+                )
+
+        logger.debug(
+            "Embeddings generated",
+            count=len(texts),
+            model=self._model_name,
+        )
+
+        return embeddings
+
+    def embed_batch(
+        self,
+        texts: list[str],
+        batch_size: int = 64,
+    ) -> list[list[float]]:
+        """Embed large list of texts in batches.
+        
+        Note: CodeEmbedder already handles batching internally,
+        so this is essentially a pass-through. Kept for interface
+        compatibility with previous NomicEmbedder.
+        
+        Args:
+            texts: List of texts to embed.
+            batch_size: Number of texts per batch (passed to CodeEmbedder).
+            
+        Returns:
+            List of embedding vectors (same order as input).
+            
+        Raises:
+            Exception: If any batch fails.
+        """
+        if not texts:
+            return []
+
+        # CodeEmbedder handles batching, but we respect the batch_size hint
+        all_embeddings = []
+
+        for i in range(0, len(texts), batch_size):
+            batch = texts[i:i + batch_size]
+            batch_embeddings = self.embed(batch)
+            all_embeddings.extend(batch_embeddings)
+
+            logger.debug(
+                "Batch processed",
+                batch_num=i // batch_size + 1,
+                batch_size=len(batch),
+                total_batches=(len(texts) + batch_size - 1) // batch_size,
+            )
+
+        return all_embeddings
+
+    def clear_cache(self) -> None:
+        """Clear the embedding cache.
+        
+        Delegates to underlying CodeEmbedder's cache clearing.
+        """
+        # CodeEmbedder uses external SQLite cache, not in-memory
+        # Cache clearing would need to be implemented at the cache layer
+        logger.warning(
+            "Cache clearing not implemented - cache is persistent SQLite",
+            hint="Use contextual.embedding.cache.EmbeddingCache.clear() if needed",
+        )
+
+
+# Factory function for creating embedder instances
+def create_embedder(**kwargs) -> EmbeddingProvider:
+    """Create embedding provider instance.
+    
+    Args:
+        **kwargs: Reserved for future extensibility (currently ignored).
+        
+    Returns:
+        JinaEmbedder instance.
+        
+    Note:
+        Previous versions supported a 'provider' parameter for switching
+        between Nomic/Jina. Now we only support Jina (fastembed).
+    """
+    # Ignore kwargs for now - only one provider (Jina)
+    if kwargs:
+        logger.warning(
+            "create_embedder() kwargs ignored",
+            kwargs=list(kwargs.keys()),
+            hint="Only Jina provider supported",
+        )
+
+    return JinaEmbedder()
+
+
+# Module-level singleton instance (lazy initialized)
+_default_embedder: JinaEmbedder | None = None
+
+
+def get_embedder() -> JinaEmbedder:
+    """Get or create default embedder instance.
+    
+    Uses singleton pattern to share embedder instance across
+    indexing pipeline. The underlying CodeEmbedder also uses
+    a singleton, so this creates a single shared model instance.
+    
+    Returns:
+        Singleton JinaEmbedder instance.
+    """
+    global _default_embedder
+    if _default_embedder is None:
+        _default_embedder = JinaEmbedder()
+        logger.info(
+            "Default embedder initialized",
+            type="JinaEmbedder",
+            singleton=True,
+        )
+    return _default_embedder
+
+
+def reset_embedder() -> None:
+    """Reset singleton embedder instance.
+    
+    Useful for testing or forcing re-initialization.
+    Not typically needed in production.
+    """
+    global _default_embedder
+    _default_embedder = None
+    logger.info("Default embedder reset")

contextual/indexing/file_watcher.py

@@ -0,0 +1,154 @@
+from __future__ import annotations
+
+import time
+from pathlib import Path
+from threading import Lock, Timer
+from typing import TYPE_CHECKING
+
+from watchdog.events import FileSystemEvent, FileSystemEventHandler
+from watchdog.observers import Observer
+
+from contextual.observability.logging import get_logger
+
+
+if TYPE_CHECKING:
+    from contextual.indexing.incremental import IncrementalIndexer
+
+logger = get_logger(__name__)
+
+
+class DebouncedFileWatcher(FileSystemEventHandler):
+    """Watches a repository directory for file changes, debounces rapid events,
+    and triggers incremental re-indexing.
+    """
+
+    def __init__(
+        self,
+        repo_root: Path,
+        incremental_indexer: IncrementalIndexer,
+        debounce_ms: int = 500,
+    ):
+        """Initialize file watcher.
+
+        Args:
+            repo_root: Repository root to watch.
+            incremental_indexer: Indexer to call on changes.
+            debounce_ms: Debounce delay in milliseconds (default 500).
+        """
+        self._repo_root = repo_root
+        self._incremental_indexer = incremental_indexer
+        self._debounce_ms = debounce_ms
+        self._observer = Observer()
+        self._pending_timers: dict[str, Timer] = {}
+        self._lock = Lock()
+
+    def start(self) -> None:
+        """Creates and starts the watchdog observer to monitor the repository.
+        """
+        self._observer.schedule(self, str(self._repo_root), recursive=True)
+        self._observer.start()
+        logger.info("File watcher started", path=str(self._repo_root))
+
+    def stop(self) -> None:
+        """Stops the file watcher and cleans up pending operations.
+        """
+        if self._observer.is_alive():
+            self._observer.stop()
+            self._observer.join()
+
+        with self._lock:
+            for timer in self._pending_timers.values():
+                timer.cancel()
+            self._pending_timers.clear()
+        logger.info("File watcher stopped")
+
+    def _schedule_index(self, file_path: Path, event_type: str) -> None:
+        """Schedules an indexing operation for a file, debouncing rapid events.
+        """
+        with self._lock:
+            # Cancel any existing timer for this file
+            if str(file_path) in self._pending_timers:
+                self._pending_timers[str(file_path)].cancel()
+
+            # Create a new timer
+            timer = Timer(
+                self._debounce_ms / 1000.0,
+                self._process_file,
+                args=[file_path, event_type],
+            )
+            self._pending_timers[str(file_path)] = timer
+            timer.start()
+
+    def _process_file(self, file_path: Path, event_type: str) -> None:
+        """Processes a single file event after the debounce period.
+        """
+        try:
+            with self._lock:
+                # Remove timer from pending list
+                self._pending_timers.pop(str(file_path), None)
+
+            if not self._incremental_indexer.should_index_file(file_path):
+                return
+
+            start_time = time.monotonic()
+            if event_type in ("modified", "created"):
+                self._incremental_indexer.handle_file_change(file_path)
+            elif event_type == "deleted":
+                self._incremental_indexer.handle_file_deletion(file_path)
+
+            duration_ms = (time.monotonic() - start_time) * 1000
+            logger.debug(
+                "Processed file event",
+                path=str(file_path),
+                event=event_type,
+                duration_ms=duration_ms,
+            )
+        except Exception:
+            logger.exception(
+                "Error processing file event", path=str(file_path), event=event_type
+            )
+
+    def on_modified(self, event: FileSystemEvent) -> None:
+        """Handle file modification events."""
+        if not event.is_directory:
+            try:
+                file_path = Path(event.src_path)
+                self._schedule_index(file_path, "modified")
+            except Exception:
+                logger.exception("Error in on_modified handler", path=event.src_path)
+
+    def on_created(self, event: FileSystemEvent) -> None:
+        """Handle file creation events."""
+        if not event.is_directory:
+            try:
+                file_path = Path(event.src_path)
+                self._schedule_index(file_path, "created")
+            except Exception:
+                logger.exception("Error in on_created handler", path=event.src_path)
+
+    def on_deleted(self, event: FileSystemEvent) -> None:
+        """Handle file deletion events."""
+        if not event.is_directory:
+            try:
+                file_path = Path(event.src_path)
+                self._schedule_index(file_path, "deleted")
+            except Exception:
+                logger.exception("Error in on_deleted handler", path=event.src_path)
+
+    def on_moved(self, event: FileSystemEvent) -> None:
+        """Handle file move/rename events."""
+        if not event.is_directory:
+            try:
+                old_path = Path(event.src_path)
+                new_path = Path(event.dest_path)
+
+                # Cancel any pending timers for the old path
+                with self._lock:
+                    if str(old_path) in self._pending_timers:
+                        self._pending_timers[str(old_path)].cancel()
+                        self._pending_timers.pop(str(old_path), None)
+
+                self._incremental_indexer.handle_file_rename(old_path, new_path)
+                logger.debug("Processed file move event", src=str(old_path), dest=str(new_path))
+            except Exception:
+                logger.exception("Error in on_moved handler", src=event.src_path, dest=event.dest_path)

contextual/indexing/incremental.py

@@ -0,0 +1,260 @@
+"""Incremental indexing with smart change detection.
+
+Handles file modifications, additions, deletions, and renames efficiently
+using content-hash comparison and mtime checks.
+"""
+
+from __future__ import annotations
+
+import hashlib
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from contextual.indexing.index_writer import IndexWriter
+from contextual.observability.logging import get_logger
+
+
+if TYPE_CHECKING:
+    from contextual.indexing.pipeline import IndexingPipeline
+
+logger = get_logger(__name__)
+
+
+def compute_file_hash(file_path: Path) -> str:
+    """Compute SHA-256 hash of file content.
+    
+    Args:
+        file_path: Path to file.
+        
+    Returns:
+        Hex digest of file hash.
+    """
+    sha256 = hashlib.sha256()
+
+    try:
+        with open(file_path, "rb") as f:
+            # Read in chunks to handle large files
+            for chunk in iter(lambda: f.read(65536), b""):
+                sha256.update(chunk)
+        return sha256.hexdigest()
+    except Exception as e:
+        logger.error(
+            "Failed to compute file hash",
+            path=str(file_path),
+            error=str(e),
+        )
+        # Return empty string to force re-index
+        return ""
+
+
+class IncrementalIndexer:
+    """Handles incremental indexing of changed files.
+    
+    Detects changes via:
+    1. Content hash comparison (primary)
+    2. Modification time check (secondary)
+    3. File existence check
+    """
+
+    def __init__(self, pipeline: IndexingPipeline):
+        """Initialize incremental indexer.
+        
+        Args:
+            pipeline: Indexing pipeline instance.
+        """
+        self.pipeline = pipeline
+        self.repo_root = pipeline.repo_root
+        self.db_pool = pipeline.db_pool
+
+    def handle_file_change(self, file_path: Path) -> dict[str, int]:
+        """Handle file modification or creation.
+        
+        Checks if file actually changed before re-indexing.
+        
+        Args:
+            file_path: Absolute path to changed file.
+            
+        Returns:
+            Stats dict from indexing.
+        """
+        # Get relative path
+        try:
+            rel_path = str(file_path.resolve().relative_to(self.repo_root.resolve()))
+        except ValueError:
+            logger.warning(
+                "File outside repo root",
+                path=str(file_path),
+                repo_root=str(self.repo_root),
+            )
+            return {"files": 0, "chunks": 0, "symbols": 0, "embeddings": 0}
+
+        # Check if file exists
+        if not file_path.exists():
+            logger.debug("File no longer exists", path=str(file_path))
+            return self.handle_file_deletion(file_path)
+
+        # Compute current file hash
+        current_hash = compute_file_hash(file_path)
+
+        # Check existing file hash in DB
+        with self.db_pool.reader() as reader_conn:
+            cursor = reader_conn.cursor()
+            cursor.execute(
+                "SELECT content_hash, mtime FROM files WHERE path = ?",
+                (rel_path,)
+            )
+            existing = cursor.fetchone()
+
+        if existing:
+            existing_hash, existing_mtime = existing
+            current_mtime = int(file_path.stat().st_mtime)
+
+            # Check if content unchanged
+            if existing_hash == current_hash:
+                logger.debug(
+                    "File content unchanged (hash match)",
+                    path=str(file_path),
+                    hash=current_hash[:8],
+                )
+
+                # Update mtime if changed (file was touched)
+                if current_mtime != existing_mtime:
+                    with self.db_pool.writer() as writer_conn:
+                        cursor = writer_conn.cursor()
+                        cursor.execute(
+                            "UPDATE files SET mtime = ? WHERE path = ?",
+                            (current_mtime, rel_path)
+                        )
+                        writer_conn.commit()
+
+                return {"files": 0, "chunks": 0, "symbols": 0, "embeddings": 0}
+
+        # File is new or changed - re-index
+        logger.info(
+            "File changed, re-indexing",
+            path=str(file_path),
+            new=existing is None,
+        )
+
+        stats = self.pipeline.index_single_file(file_path)
+
+        return stats
+
+    def handle_file_deletion(self, file_path: Path) -> dict[str, int]:
+        """Handle file deletion.
+        
+        Args:
+            file_path: Absolute path to deleted file.
+            
+        Returns:
+            Stats dict with chunks_deleted count.
+        """
+        logger.info("File deleted", path=str(file_path))
+
+        chunks_deleted = self.pipeline.delete_file(file_path)
+
+        return {
+            "files": -1,
+            "chunks": -chunks_deleted,
+            "symbols": 0,  # Cascade delete handled by DB
+            "embeddings": 0,
+        }
+
+    def handle_file_rename(
+        self,
+        old_path: Path,
+        new_path: Path,
+    ) -> dict[str, int]:
+        """Handle file rename/move.
+        
+        Deletes old entry and creates new one.
+        
+        Args:
+            old_path: Old absolute path.
+            new_path: New absolute path.
+            
+        Returns:
+            Combined stats dict.
+        """
+        logger.info(
+            "File renamed",
+            old_path=str(old_path),
+            new_path=str(new_path),
+        )
+
+        # Delete old entry
+        delete_stats = self.handle_file_deletion(old_path)
+
+        # Index new entry
+        add_stats = self.handle_file_change(new_path)
+
+        # Combine stats
+        return {
+            "files": delete_stats["files"] + add_stats["files"],
+            "chunks": delete_stats["chunks"] + add_stats["chunks"],
+            "symbols": delete_stats["symbols"] + add_stats["symbols"],
+            "embeddings": delete_stats["embeddings"] + add_stats["embeddings"],
+        }
+
+    def update_indexing_state(self, stats: dict[str, int]) -> None:
+        """Update incremental indexing state.
+        
+        Args:
+            stats: Stats from incremental indexing.
+        """
+        # Get current totals from DB
+        with self.db_pool.reader() as reader_conn:
+            cursor = reader_conn.cursor()
+
+            cursor.execute("SELECT COUNT(*) FROM files")
+            total_files = cursor.fetchone()[0]
+
+            cursor.execute("SELECT COUNT(*) FROM chunks")
+            total_chunks = cursor.fetchone()[0]
+
+            cursor.execute("SELECT COUNT(*) FROM code_symbols")
+            total_symbols = cursor.fetchone()[0]
+
+        # Update state
+        with self.db_pool.writer() as writer_conn:
+            writer = IndexWriter(writer_conn)
+            writer.update_indexing_state(
+                repo_root=str(self.repo_root),
+                total_files=total_files,
+                total_chunks=total_chunks,
+                total_symbols=total_symbols,
+                full_index=False,  # Incremental
+            )
+
+    def should_index_file(self, file_path: Path) -> bool:
+        """Check if file should be indexed.
+        
+        Uses FileProcessor's filtering logic.
+        
+        Args:
+            file_path: Path to check.
+            
+        Returns:
+            True if file should be indexed.
+        """
+        # Check if file exists
+        if not file_path.exists() or not file_path.is_file():
+            return False
+
+        # Get relative path
+        try:
+            rel_path = file_path.resolve().relative_to(self.repo_root.resolve())
+        except ValueError:
+            return False
+
+        # Check if should skip
+        if self.pipeline.file_processor._should_skip(file_path, rel_path):
+            return False
+
+        # Check if language supported
+        from contextual.indexing.chunker import detect_language
+        language = detect_language(file_path)
+        if not language:
+            return False
+
+        return True