flurryx-code-memory 0.4.0__py3-none-any.whl

code_memory/sync/watcher.py

@@ -0,0 +1,320 @@
+"""Cross-platform filesystem watcher.
+
+Uses ``watchdog`` (FSEvents on macOS, inotify on Linux, ReadDirectoryChangesW
+on Windows) when available; otherwise falls back to mtime polling so the
+feature degrades gracefully even when the optional dep is missing.
+
+The watcher debounces bursts (e.g. ``git checkout`` touches many files at
+once) and triggers a single ``sync_repo`` per quiet period. Excluded paths
+include ``.git`` and any directory the extractor's gitignore filter drops.
+"""
+
+from __future__ import annotations
+
+import logging
+import threading
+import time
+from collections.abc import Callable
+from pathlib import Path
+
+from .sync import SyncResult, sync_repo
+
+log = logging.getLogger("codememory.watcher")
+
+DEFAULT_DEBOUNCE = 2.0
+DEFAULT_POLL_INTERVAL = 5.0
+
+ExcludeFn = Callable[[Path], bool]
+
+
+EXCLUDED_ROOT_DIRS: tuple[str, ...] = (
+    # VCS / project state
+    ".git",
+    "data",
+    # Virtualenvs / package roots
+    ".venv",
+    "node_modules",
+    # Build outputs
+    "dist",
+    "out-tsc",
+    "build",
+    "target",
+    "coverage",
+    # Framework / bundler caches
+    ".angular",
+    ".next",
+    ".nuxt",
+    ".svelte-kit",
+    ".turbo",
+    ".parcel-cache",
+    ".cache",
+    # Python caches
+    "__pycache__",
+    ".pytest_cache",
+    ".mypy_cache",
+    ".ruff_cache",
+    ".tox",
+    # Editor metadata
+    ".idea",
+    ".vscode",
+    # Agentic tool caches (high write churn, no source value)
+    ".opencode",
+    ".serena",
+    ".claude",
+    ".cursor",
+    ".windsurf",
+    ".clavix",
+)
+
+
+def _default_exclude(repo: Path) -> ExcludeFn:
+    repo_root = repo.resolve()
+    blocked = tuple((repo_root / name).resolve() for name in EXCLUDED_ROOT_DIRS)
+
+    def exclude(p: Path) -> bool:
+        try:
+            r = p.resolve()
+        except OSError:
+            return True
+        for b in blocked:
+            try:
+                r.relative_to(b)
+                return True
+            except ValueError:
+                continue
+        return False
+
+    return exclude
+
+
+class Debouncer:
+    """Coalesce bursts; fire ``flush`` once activity quiets for ``window`` seconds."""
+
+    def __init__(
+        self,
+        window: float,
+        flush: Callable[[], None],
+    ) -> None:
+        self.window = window
+        self.flush = flush
+        self._lock = threading.Lock()
+        self._timer: threading.Timer | None = None
+        self._dirty = False
+
+    def bump(self) -> None:
+        with self._lock:
+            self._dirty = True
+            if self._timer is not None:
+                self._timer.cancel()
+            self._timer = threading.Timer(self.window, self._fire)
+            self._timer.daemon = True
+            self._timer.start()
+
+    def _fire(self) -> None:
+        with self._lock:
+            if not self._dirty:
+                return
+            self._dirty = False
+        try:
+            self.flush()
+        except Exception:  # noqa: BLE001
+            log.exception("debounced flush failed")
+
+    def cancel(self) -> None:
+        with self._lock:
+            if self._timer is not None:
+                self._timer.cancel()
+                self._timer = None
+
+
+class Watcher:
+    """Long-running watcher for a single repo."""
+
+    def __init__(
+        self,
+        repo: Path,
+        *,
+        project: str | None = None,
+        debounce: float = DEFAULT_DEBOUNCE,
+        poll_interval: float = DEFAULT_POLL_INTERVAL,
+        on_sync: Callable[[SyncResult], None] | None = None,
+    ) -> None:
+        self.repo = Path(repo).resolve()
+        self.project = project
+        self.debounce_window = debounce
+        self.poll_interval = poll_interval
+        self.on_sync = on_sync
+        self.exclude = _default_exclude(self.repo)
+        self._stop = threading.Event()
+        self._thread: threading.Thread | None = None
+        self._debouncer = Debouncer(debounce, self._trigger_sync)
+
+    # ------------------------------------------------------------------
+    # Lifecycle
+    # ------------------------------------------------------------------
+
+    def start(self, *, blocking: bool = False) -> None:
+        if self._thread is not None and self._thread.is_alive():
+            return
+        self._stop.clear()
+        target = self._run_watchdog if _watchdog_available() else self._run_poll
+        if blocking:
+            target()
+            return
+        self._thread = threading.Thread(target=target, name="cm-watch", daemon=True)
+        self._thread.start()
+
+    def stop(self) -> None:
+        self._stop.set()
+        self._debouncer.cancel()
+        if self._thread is not None:
+            self._thread.join(timeout=5)
+            self._thread = None
+
+    def is_running(self) -> bool:
+        return self._thread is not None and self._thread.is_alive()
+
+    # ------------------------------------------------------------------
+    # Implementations
+    # ------------------------------------------------------------------
+
+    def _run_watchdog(self) -> None:
+        from watchdog.events import FileSystemEventHandler
+        from watchdog.observers import Observer
+
+        watcher = self
+
+        class _Handler(FileSystemEventHandler):
+            def on_any_event(self, event):  # noqa: ANN001 - lib type
+                if event.is_directory:
+                    return
+                path = Path(getattr(event, "dest_path", None) or event.src_path)
+                watcher._handle_path(path)
+
+        observer = Observer()
+        observer.schedule(_Handler(), str(self.repo), recursive=True)
+        observer.start()
+        log.info("watcher started (watchdog) on %s", self.repo)
+        try:
+            while not self._stop.wait(0.5):
+                pass
+        finally:
+            observer.stop()
+            observer.join(timeout=3)
+            log.info("watcher stopped (watchdog)")
+
+    def _run_poll(self) -> None:
+        log.info(
+            "watchdog not installed; falling back to mtime polling on %s "
+            "(install `watchdog` for native events)",
+            self.repo,
+        )
+        last_mtime = self._max_mtime()
+        last_head = self._git_head()
+        while not self._stop.wait(self.poll_interval):
+            mtime = self._max_mtime()
+            head = self._git_head()
+            if mtime != last_mtime or head != last_head:
+                last_mtime = mtime
+                last_head = head
+                self._debouncer.bump()
+
+    def _max_mtime(self) -> float:
+        latest = 0.0
+        for root, dirs, files in _safe_walk(self.repo):
+            r = Path(root)
+            dirs[:] = [d for d in dirs if not self.exclude(r / d)]
+            for name in files:
+                p = r / name
+                if self.exclude(p):
+                    continue
+                try:
+                    mt = p.stat().st_mtime
+                except OSError:
+                    continue
+                if mt > latest:
+                    latest = mt
+        return latest
+
+    def _git_head(self) -> str | None:
+        try:
+            from ..orchestrator import git_delta
+
+            return git_delta.head_sha(self.repo) if git_delta.is_git_repo(self.repo) else None
+        except Exception:  # noqa: BLE001
+            return None
+
+    # ------------------------------------------------------------------
+    # Event routing
+    # ------------------------------------------------------------------
+
+    def _is_ref_event(self, path: Path) -> bool:
+        """True when ``path`` is a git ref whose change should re-sync."""
+        try:
+            rel = path.resolve().relative_to(self.repo)
+        except (OSError, ValueError):
+            return False
+        parts = rel.parts
+        if not parts or parts[0] != ".git":
+            return False
+        if parts == (".git", "HEAD"):
+            return True
+        if len(parts) >= 4 and parts[1:3] == ("refs", "heads"):
+            return True
+        return False
+
+    def _handle_path(self, path: Path) -> None:
+        """Decide whether ``path`` should trigger a debounced sync."""
+        if self._is_ref_event(path):
+            self._debouncer.bump()
+            return
+        if self.exclude(path):
+            return
+        self._debouncer.bump()
+
+    # ------------------------------------------------------------------
+    # Sync
+    # ------------------------------------------------------------------
+
+    def _trigger_sync(self) -> None:
+        log.debug("watcher firing sync for %s", self.repo)
+        try:
+            result = sync_repo(self.repo, project=self.project, trigger="watcher")
+        except Exception:  # noqa: BLE001
+            log.exception("watcher sync failed")
+            return
+        if self.on_sync:
+            try:
+                self.on_sync(result)
+            except Exception:  # noqa: BLE001
+                log.exception("on_sync callback raised")
+
+
+def _watchdog_available() -> bool:
+    try:
+        import watchdog  # noqa: F401
+        import watchdog.observers  # noqa: F401
+
+        return True
+    except ImportError:
+        return False
+
+
+def _safe_walk(root: Path):
+    import os
+
+    for entry in os.walk(root):
+        yield entry
+
+
+def run_foreground(repo: Path, *, project: str | None = None) -> None:
+    """Blocking CLI entry: start the watcher until Ctrl-C."""
+    w = Watcher(repo, project=project)
+    w.start(blocking=False)
+    try:
+        while True:
+            time.sleep(1)
+    except KeyboardInterrupt:
+        pass
+    finally:
+        w.stop()

code_memory/vector/__init__.py

@@ -0,0 +1,3 @@
+from .qdrant_store import QdrantStore, VectorHit, VectorRecord
+
+__all__ = ["QdrantStore", "VectorHit", "VectorRecord"]

code_memory/vector/qdrant_store.py

@@ -0,0 +1,302 @@
+from __future__ import annotations
+
+from collections.abc import Iterable, Sequence
+from dataclasses import dataclass, field
+from typing import Any
+
+from qdrant_client import QdrantClient
+from qdrant_client.http import models as qm
+
+from ..config import CONFIG
+from ..embed import HybridVec
+
+# Vector slot names inside each Qdrant point. Keep stable; collection
+# rebuild is required to change them.
+DENSE = "dense"
+SPARSE = "sparse"
+
+
+@dataclass
+class VectorRecord:
+    id: str
+    vector: HybridVec
+    payload: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class VectorHit:
+    id: str
+    score: float
+    payload: dict[str, Any]
+
+
+class QdrantStore:
+    """Hybrid dense + sparse store w/ server-side RRF fusion.
+
+    Collections use Qdrant's named-vector layout: each point carries a
+    ``dense`` slot (m3 1024-d cosine) and a ``sparse`` slot (m3 lexical
+    weights, IDF-modified). Queries prefetch both, then fuse with
+    Reciprocal Rank Fusion so neither view dominates on its own.
+    """
+
+    def __init__(
+        self,
+        url: str | None = None,
+        dim: int | None = None,
+    ) -> None:
+        from ..config import resolve_embed_dim
+
+        self.url = url or CONFIG.qdrant_url
+        self.client = QdrantClient(url=self.url)
+        # ``CONFIG.embed_dim`` is 0 by default (sentinel for "auto").
+        # Resolve via the known-model table so ``EMBED_MODEL``
+        # automatically picks the right dim without the operator setting
+        # ``EMBED_DIM``. Explicit ``dim`` arg or ``EMBED_DIM`` env
+        # still wins.
+        self.dim = (
+            dim
+            if dim is not None
+            else resolve_embed_dim(CONFIG.embed_model, CONFIG.embed_dim)
+        )
+
+    # --------------------------------------------------------- collection
+
+    def ensure_collection(self, name: str) -> None:
+        status = self._inspect_collection(name)
+        if status == "hybrid":
+            # Check dimension match so mismatched embedding models are
+            # caught early with a clear error rather than cryptic Qdrant
+            # gRPC failures at upsert time.
+            existing = self.client.get_collection(collection_name=name)
+            vectors = getattr(existing.config.params, "vectors", None)
+            if isinstance(vectors, dict) and DENSE in vectors:
+                existing_dim = getattr(vectors[DENSE], "size", None)
+                if existing_dim is not None and existing_dim != self.dim:
+                    raise ValueError(
+                        f"Collection '{name}' exists with dimension {existing_dim:,}d, "
+                        f"but embedding model produces {self.dim:,}d. "
+                        f"Re-ingest (code-memory ingest --full) or delete the collection and re-create."
+                    )
+            return
+        if status == "legacy":
+            # Caller is on the ingest path and explicitly asked us to make
+            # the collection ready — drop the legacy layout and recreate.
+            # Read paths never trigger this branch because they go through
+            # ``_inspect_collection`` directly.
+            try:
+                self.client.delete_collection(collection_name=name)
+            except Exception:  # noqa: BLE001
+                pass
+        self._create_hybrid(name)
+
+    def recreate_collection(self, name: str) -> None:
+        """Drop and re-create. Used by full re-ingest."""
+        try:
+            self.client.delete_collection(collection_name=name)
+        except Exception:
+            pass
+        self._create_hybrid(name)
+
+    def _inspect_collection(self, name: str) -> str:
+        """Pure read of the collection's schema. No side effects.
+
+        Returns ``"missing"``, ``"legacy"`` (single-vector layout left
+        over from before the hybrid migration), or ``"hybrid"``.
+        """
+        existing = {c.name for c in self.client.get_collections().collections}
+        if name not in existing:
+            return "missing"
+        info = self.client.get_collection(collection_name=name)
+        vectors = getattr(info.config.params, "vectors", None)
+        sparse = getattr(info.config.params, "sparse_vectors", None)
+        has_dense = isinstance(vectors, dict) and DENSE in vectors
+        has_sparse = isinstance(sparse, dict) and SPARSE in sparse
+        if has_dense and has_sparse:
+            return "hybrid"
+        return "legacy"
+
+    def _create_hybrid(self, name: str) -> None:
+        self.client.create_collection(
+            collection_name=name,
+            vectors_config={
+                DENSE: qm.VectorParams(size=self.dim, distance=qm.Distance.COSINE),
+            },
+            sparse_vectors_config={
+                SPARSE: qm.SparseVectorParams(
+                    modifier=qm.Modifier.IDF,
+                ),
+            },
+        )
+
+    # ------------------------------------------------------------- upsert
+
+    def upsert(self, collection: str, records: Iterable[VectorRecord]) -> None:
+        points: list[qm.PointStruct] = []
+        for r in records:
+            vec_payload: dict[str, Any] = {DENSE: r.vector.dense}
+            # Skip the sparse slot when the embedder didn't emit one
+            # (Ollama backend returns ``HybridVec`` with empty sparse).
+            # Qdrant rejects sparse vectors with zero indices.
+            if r.vector.sparse.indices:
+                vec_payload[SPARSE] = qm.SparseVector(
+                    indices=r.vector.sparse.indices,
+                    values=r.vector.sparse.values,
+                )
+            points.append(
+                qm.PointStruct(id=r.id, vector=vec_payload, payload=r.payload)
+            )
+        if not points:
+            return
+        self.client.upsert(collection_name=collection, points=points)
+
+    # ------------------------------------------------------------- search
+
+    def search(
+        self,
+        collection: str,
+        vector: HybridVec | Sequence[float],
+        top_k: int = 10,
+        filt: dict[str, Any] | None = None,
+        *,
+        prefetch_multiplier: int = 4,
+        mode: str = "hybrid",
+    ) -> list[VectorHit]:
+        """Hybrid search with RRF fusion.
+
+        ``vector`` may be a :class:`HybridVec` (preferred) for full
+        dense+sparse fusion, or a plain dense sequence for backwards
+        compatibility with legacy callers / tests. Sparse-less queries
+        degrade gracefully to a dense-only ranking.
+
+        ``prefetch_multiplier`` controls how many candidates each branch
+        pulls before fusion. 4x is the Qdrant docs default and gives
+        enough overlap for RRF to do useful work.
+
+        ``mode`` is an A/B test seam used by the benchmark harness:
+        ``"hybrid"`` (default) fuses both vectors; ``"dense"`` ignores
+        the sparse slot entirely. Production callers should leave it at
+        the default — query-time degradation is for measurement only.
+        """
+        status = self._inspect_collection(collection)
+        if status == "missing":
+            raise LookupError(
+                f"Qdrant collection '{collection}' does not exist. "
+                f"Run `code-memory ingest <path> --project <slug>` first."
+            )
+        if status == "legacy":
+            raise RuntimeError(
+                f"Qdrant collection '{collection}' uses the legacy "
+                f"single-vector layout from before the hybrid migration. "
+                f"Drop it and re-ingest:\n"
+                f"  curl -X DELETE {self.url}/collections/{collection}\n"
+                f"  code-memory ingest <path> --full"
+            )
+        qfilter = _to_filter(filt) if filt else None
+
+        # Hybrid mode requires a non-empty sparse query vector. When the
+        # embedder is dense-only (Ollama backend), fall through to the
+        # dense path so callers don't need to special-case the backend.
+        hv = vector if isinstance(vector, HybridVec) else None
+        has_sparse = hv is not None and bool(hv.sparse.indices)
+        if hv is not None and has_sparse and mode in ("hybrid", "hybrid_dbsf"):
+            prefetch = [
+                qm.Prefetch(
+                    query=hv.dense,
+                    using=DENSE,
+                    limit=top_k * prefetch_multiplier,
+                    filter=qfilter,
+                ),
+                qm.Prefetch(
+                    query=qm.SparseVector(
+                        indices=hv.sparse.indices,
+                        values=hv.sparse.values,
+                    ),
+                    using=SPARSE,
+                    limit=top_k * prefetch_multiplier,
+                    filter=qfilter,
+                ),
+            ]
+            fusion = qm.Fusion.DBSF if mode == "hybrid_dbsf" else qm.Fusion.RRF
+            res = self.client.query_points(
+                collection_name=collection,
+                prefetch=prefetch,
+                query=qm.FusionQuery(fusion=fusion),
+                limit=top_k,
+                with_payload=True,
+                query_filter=qfilter,
+            )
+        else:
+            # Dense-only path: legacy callers + benchmark "dense" mode
+            dense_vec = vector.dense if isinstance(vector, HybridVec) else list(vector)
+            res = self.client.query_points(
+                collection_name=collection,
+                query=dense_vec,
+                using=DENSE,
+                limit=top_k,
+                query_filter=qfilter,
+                with_payload=True,
+            )
+
+        return [
+            VectorHit(id=str(p.id), score=float(p.score), payload=p.payload or {})
+            for p in res.points
+        ]
+
+    def delete_by_path(self, collection: str, path: str) -> None:
+        self.client.delete(
+            collection_name=collection,
+            points_selector=qm.FilterSelector(
+                filter=qm.Filter(
+                    must=[qm.FieldCondition(key="path", match=qm.MatchValue(value=path))]
+                )
+            ),
+        )
+
+    def delete_by_ids(self, collection: str, ids: Sequence[str]) -> None:
+        """Bulk delete points by id. No-op on empty input."""
+        if not ids:
+            return
+        self.client.delete(
+            collection_name=collection,
+            points_selector=qm.PointIdsList(points=list(ids)),
+        )
+
+    def set_payload(
+        self,
+        collection: str,
+        ids: Sequence[str],
+        payload: dict[str, Any],
+    ) -> None:
+        """Merge ``payload`` into points identified by ``ids``.
+
+        Used by the claim indexer to flip ``open`` from ``True`` to
+        ``False`` when a claim is superseded, without re-embedding the
+        triple. No-op on empty ids — Qdrant rejects empty selectors.
+        """
+        if not ids:
+            return
+        self.client.set_payload(
+            collection_name=collection,
+            payload=payload,
+            points=list(ids),
+        )
+
+    def count(self, collection: str) -> int:
+        """Return total point count for the collection.
+
+        Returns ``0`` for missing collections so callers can use this
+        as a cheap "do I need to backfill?" probe without try/except
+        around ``ensure_collection``.
+        """
+        if self._inspect_collection(collection) == "missing":
+            return 0
+        res = self.client.count(collection_name=collection, exact=False)
+        return int(res.count)
+
+
+def _to_filter(filt: dict[str, Any]) -> qm.Filter:
+    must = [
+        qm.FieldCondition(key=k, match=qm.MatchValue(value=v))
+        for k, v in filt.items()
+    ]
+    return qm.Filter(must=must)

flurryx_code_memory-0.4.0.dist-info/METADATA

@@ -0,0 +1,26 @@
+Metadata-Version: 2.4
+Name: flurryx-code-memory
+Version: 0.4.0
+Summary: Local lightweight memory layer for coding agents: FalkorDB + Qdrant + Ollama (BGE-M3) + tree-sitter
+Requires-Python: >=3.11
+Requires-Dist: anyio>=4.4
+Requires-Dist: falkordb>=1.0.10
+Requires-Dist: httpx>=0.27
+Requires-Dist: mcp>=1.0
+Requires-Dist: pydantic>=2.8
+Requires-Dist: qdrant-client>=1.12
+Requires-Dist: rich>=13.7
+Requires-Dist: tree-sitter-language-pack>=0.7
+Requires-Dist: tree-sitter>=0.23
+Requires-Dist: typer>=0.12
+Requires-Dist: watchdog>=4.0
+Provides-Extra: dev
+Requires-Dist: mypy>=1.11; extra == 'dev'
+Requires-Dist: pytest-asyncio>=0.24; extra == 'dev'
+Requires-Dist: pytest>=8.3; extra == 'dev'
+Requires-Dist: ruff>=0.6; extra == 'dev'
+Provides-Extra: dotnet
+Requires-Dist: dnfile>=0.15; extra == 'dotnet'
+Provides-Extra: hybrid
+Requires-Dist: flagembedding>=1.3; extra == 'hybrid'
+Requires-Dist: torch>=2.1; extra == 'hybrid'