contextual-engine 0.1.0__py3-none-any.whl

contextual/docs/watcher.py

@@ -0,0 +1,265 @@
+"""Docs file watcher for Contextual Phase 2.
+
+Extends the Phase 1 watchdog integration to handle markdown/RST/txt files.
+Routes filesystem events to ``DocsPipeline.index_file()`` or
+``DocsPipeline.delete_file()`` with 400ms debounce to absorb editor
+atomic-write sequences (Vim, VS Code, JetBrains all do write-rename).
+
+Design:
+- Mirrors the existing ``contextual/indexing/file_watcher.py`` pattern exactly.
+- Composable: the MCP server runs BOTH watchers from a single background
+  thread pool (one Observer, two Handlers scheduled on the same path).
+- Never blocks the main thread. All indexing happens on a daemon thread pool.
+- Logging to stderr via structlog (never stdout — MCP stdio transport).
+
+Vim/JetBrains atomic-rename caveat:
+  These editors write to a temp file then rename to the target.  Watchdog
+  fires ``FileMovedEvent`` (not ``FileModifiedEvent``).  We handle BOTH
+  ``on_modified`` and ``on_moved`` (dest path) so no event is missed.
+"""
+from __future__ import annotations
+
+import threading
+import time
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+import structlog
+from watchdog.events import (
+    FileCreatedEvent,
+    FileDeletedEvent,
+    FileModifiedEvent,
+    FileMovedEvent,
+    FileSystemEvent,
+    PatternMatchingEventHandler,
+)
+from watchdog.observers import Observer
+
+if TYPE_CHECKING:
+    from contextual.docs.pipeline import DocsPipeline
+
+logger = structlog.get_logger(__name__)
+
+# ---------------------------------------------------------------------------
+# Constants
+# ---------------------------------------------------------------------------
+
+DOC_PATTERNS: list[str] = ["*.md", "*.mdx", "*.rst", "*.txt"]
+DOC_EXTENSIONS: frozenset[str] = frozenset({".md", ".mdx", ".rst", ".txt"})
+
+# Debounce window: editor atomic-write sequences complete within 200ms;
+# 400ms gives a comfortable margin on slow NFS/network mounts.
+DEBOUNCE_SECONDS: float = 0.4
+
+
+# ---------------------------------------------------------------------------
+# Event handler
+# ---------------------------------------------------------------------------
+
+class _DocsEventHandler(PatternMatchingEventHandler):
+    """Watchdog event handler for documentation files.
+
+    Debounces rapid successive events for the same file and dispatches
+    indexing work to a background thread pool.
+    """
+
+    def __init__(self, pipeline: "DocsPipeline") -> None:
+        super().__init__(
+            patterns=DOC_PATTERNS,
+            ignore_directories=True,
+            case_sensitive=False,
+        )
+        self._pipeline = pipeline
+        self._lock = threading.Lock()
+        # Pending timers keyed by absolute file path string
+        self._timers: dict[str, threading.Timer] = {}
+
+    # ------------------------------------------------------------------
+    # Watchdog callbacks
+    # ------------------------------------------------------------------
+
+    def on_created(self, event: FileSystemEvent) -> None:
+        self._schedule(Path(event.src_path), "created")
+
+    def on_modified(self, event: FileSystemEvent) -> None:
+        self._schedule(Path(event.src_path), "modified")
+
+    def on_deleted(self, event: FileSystemEvent) -> None:
+        self._schedule_delete(Path(event.src_path))
+
+    def on_moved(self, event: FileMovedEvent) -> None:  # type: ignore[override]
+        # Old path → delete; new path → index (if doc extension)
+        self._schedule_delete(Path(event.src_path))
+        dest = Path(event.dest_path)
+        if dest.suffix.lower() in DOC_EXTENSIONS:
+            self._schedule(dest, "moved")
+
+    # ------------------------------------------------------------------
+    # Debounce helpers
+    # ------------------------------------------------------------------
+
+    def _schedule(self, path: Path, event_type: str) -> None:
+        """Debounce an index event for ``path``."""
+        key = str(path.resolve())
+        with self._lock:
+            # Cancel any pending timer for this path
+            if key in self._timers:
+                self._timers[key].cancel()
+            timer = threading.Timer(
+                DEBOUNCE_SECONDS,
+                self._run_index,
+                args=(path, event_type),
+            )
+            timer.daemon = True
+            self._timers[key] = timer
+        timer.start()
+        logger.debug("Docs event debounced", path=str(path), event=event_type)
+
+    def _schedule_delete(self, path: Path) -> None:
+        """Debounce a delete event for ``path``."""
+        key = str(path.resolve())
+        with self._lock:
+            if key in self._timers:
+                self._timers[key].cancel()
+            timer = threading.Timer(
+                DEBOUNCE_SECONDS,
+                self._run_delete,
+                args=(path,),
+            )
+            timer.daemon = True
+            self._timers[key] = timer
+        timer.start()
+
+    def _run_index(self, path: Path, event_type: str) -> None:
+        """Execute index_file on the pipeline (runs on timer thread)."""
+        with self._lock:
+            self._timers.pop(str(path.resolve()), None)
+
+        if not path.exists():
+            logger.debug("File gone before index — treating as delete", path=str(path))
+            self._run_delete(path)
+            return
+
+        t0 = time.perf_counter()
+        try:
+            stats = self._pipeline.index_file(path)
+            elapsed = time.perf_counter() - t0
+            logger.info(
+                "Docs file re-indexed",
+                path=str(path),
+                event=event_type,
+                chunks=stats.chunks_created,
+                elapsed_ms=round(elapsed * 1000, 1),
+            )
+        except Exception as exc:  # noqa: BLE001
+            logger.error("Docs incremental index failed", path=str(path), error=str(exc))
+
+    def _run_delete(self, path: Path) -> None:
+        """Execute delete_file on the pipeline (runs on timer thread)."""
+        with self._lock:
+            self._timers.pop(str(path.resolve()), None)
+        try:
+            deleted = self._pipeline.delete_file(path)
+            if deleted:
+                logger.info("Docs file removed from index", path=str(path), chunks=deleted)
+        except Exception as exc:  # noqa: BLE001
+            logger.error("Docs delete failed", path=str(path), error=str(exc))
+
+
+# ---------------------------------------------------------------------------
+# Public watcher class
+# ---------------------------------------------------------------------------
+
+class DocsFileWatcher:
+    """Watches a repository for documentation file changes.
+
+    Runs a watchdog ``Observer`` in a background daemon thread.
+    Composable with the existing ``FileWatcher`` (code watcher) — both
+    can share the same ``Observer`` instance by calling ``attach(observer)``.
+
+    Typical usage (standalone)::
+
+        watcher = DocsFileWatcher(repo_root, docs_pipeline)
+        watcher.start()
+        # ... server runs ...
+        watcher.stop()
+
+    Composing with the existing code watcher::
+
+        observer = Observer()
+        code_watcher.attach(observer)     # Phase 1 watcher registers its handler
+        docs_watcher.attach(observer)     # Phase 2 watcher registers its handler
+        observer.start()
+        # ...
+        observer.stop()
+        observer.join()
+    """
+
+    def __init__(self, repo_root: Path, pipeline: "DocsPipeline") -> None:
+        self._root = repo_root.resolve()
+        self._handler = _DocsEventHandler(pipeline)
+        self._observer: Observer | None = None
+        self._started = False
+
+    # ------------------------------------------------------------------
+    # Lifecycle
+    # ------------------------------------------------------------------
+
+    def start(self) -> None:
+        """Start the file watcher in its own Observer thread."""
+        if self._started:
+            logger.warning("DocsFileWatcher already started")
+            return
+        observer = Observer()
+        self.attach(observer)
+        observer.start()
+        self._observer = observer
+        self._started = True
+        logger.info(
+            "DocsFileWatcher started",
+            root=str(self._root),
+            patterns=DOC_PATTERNS,
+            debounce_ms=int(DEBOUNCE_SECONDS * 1000),
+        )
+
+    def stop(self) -> None:
+        """Stop the file watcher and join the Observer thread."""
+        if not self._started or self._observer is None:
+            return
+        # Cancel all pending debounce timers before stopping (W2: Fix memory leak)
+        with self._handler._lock:
+            for timer in self._handler._timers.values():
+                timer.cancel()
+            self._handler._timers.clear()
+        self._observer.stop()
+        self._observer.join(timeout=5.0)
+        self._observer = None
+        self._started = False
+        logger.info("DocsFileWatcher stopped")
+
+    def attach(self, observer: Observer) -> None:
+        """Register this handler on an external Observer instance.
+
+        Use this when composing with the Phase 1 code watcher so both
+        handlers share a single OS-level inotify/FSEvents subscription.
+
+        Args:
+            observer: A watchdog ``Observer`` that has NOT been started yet.
+        """
+        observer.schedule(
+            self._handler,
+            str(self._root),
+            recursive=True,
+        )
+        logger.debug("DocsFileWatcher attached to observer", root=str(self._root))
+
+    # ------------------------------------------------------------------
+    # Context manager support
+    # ------------------------------------------------------------------
+
+    def __enter__(self) -> "DocsFileWatcher":
+        self.start()
+        return self
+
+    def __exit__(self, *_: object) -> None:
+        self.stop()

contextual/embedding/__init__.py

@@ -0,0 +1,87 @@
+"""Embedding module: fastembed ONNX inference with content-hash caching.
+
+This module provides efficient code embedding generation using:
+- Model: jina-embeddings-v2-base-code (768-dim, int8 ONNX)
+- Caching: Content-hash based deduplication
+- Batching: Up to 64 documents per batch
+- Thread-safety: Singleton embedder with locks
+
+Public API (High-level - recommended):
+- embed_chunks_with_cache: Batch embed chunks with cache lookup/write
+- embed_query_simple: Embed search query (no cache)
+- get_embedding_stats: Cache statistics and efficiency metrics
+- warm_cache_from_chunks: Pre-populate cache from database
+
+Public API (Low-level - advanced use):
+- get_embedder: Access embedder singleton directly
+- CodeEmbedder: Embedder class (use singleton via get_embedder)
+- initialize_cache: Setup cache table
+- get_cached_embedding: Single cache lookup
+- put_cached_embedding: Single cache write
+
+Performance characteristics:
+- Model load: ~2s (one-time, cached)
+- Embedding generation: ~55ms per chunk
+- Cache hit: ~0.1ms (550x faster)
+- Batch of 64: ~3.5s total (~18 docs/sec)
+
+Example usage:
+    >>> from contextual.embedding import embed_chunks_with_cache, embed_query_simple
+    >>> from contextual.storage import SQLitePool
+    >>>
+    >>> # Indexing workflow
+    >>> pool = SQLitePool(db_path)
+    >>> chunks = [("hash1", "def parse():"), ("hash2", "class Config:")]
+    >>> with pool.writer() as conn:
+    ...     results = embed_chunks_with_cache(conn, chunks)
+    >>>
+    >>> # Search workflow
+    >>> query_vector = embed_query_simple("parse JSON configuration")
+"""
+from __future__ import annotations
+
+# High-level API (recommended)
+from contextual.embedding.helpers import (
+    embed_chunks,
+    embed_query_simple,
+    get_embedding_stats,
+    warm_cache_from_chunks,
+)
+
+# Core embedder
+from contextual.embedding.embedder import (
+    CodeEmbedder,
+    DocsEmbedder,
+    get_embedder,
+    get_docs_embedder,
+    reset_embedder,
+)
+
+# Cache operations
+from contextual.embedding.cache import (
+    clear_cache,
+    get_cache_stats,
+    get_cached_embedding,
+    initialize_cache,
+    put_cached_embedding,
+)
+
+__all__ = [
+    # High-level API (use these)
+    "embed_chunks",
+    "embed_query_simple",
+    "get_embedding_stats",
+    "warm_cache_from_chunks",
+    # Core embedder
+    "CodeEmbedder",
+    "DocsEmbedder",
+    "get_embedder",
+    "get_docs_embedder",
+    "reset_embedder",
+    # Cache operations
+    "clear_cache",
+    "get_cache_stats",
+    "get_cached_embedding",
+    "initialize_cache",
+    "put_cached_embedding",
+]