flurryx-code-memory 0.4.0__py3-none-any.whl

code_memory/orchestrator/retrieve.py

@@ -0,0 +1,505 @@
+from __future__ import annotations
+
+import logging
+import math
+import re
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+import os
+
+from ..claims import ClaimRecord, ClaimsIndexer, ClaimsStore, make_claims_indexer
+from ..metrics import MetricsStore, RetrieveTiming
+from ..config import CONFIG, Config, detect_project_slug
+from ..embed import HybridVec, M3Embedder, get_embedder
+from ..episodic import Episode, EpisodicStore
+from ..vector import QdrantStore, VectorHit
+# Hybrid (dense + sparse RRF) is opt-in. Benchmarks on the sample-webapp
+# Angular corpus showed dense-only m3 outperforms hybrid on natural
+# language queries (see docs/BENCHMARK.md). Sparse over-promotes spec
+# files and generated API stubs whose identifier vocabulary overlaps
+# heavily with the query. The collection still stores both vectors so
+# users can toggle at query time without re-ingesting.
+ENV_HYBRID = "CODEMEMORY_HYBRID"
+ENV_METRICS = "CODEMEMORY_METRICS_DB"
+
+log = logging.getLogger(__name__)
+
+
+def _retrieval_mode() -> str:
+    raw = os.environ.get(ENV_HYBRID, "0").strip().lower()
+    return "hybrid" if raw in ("1", "true", "on", "yes") else "dense"
+
+# Per-hit score adjustments applied after Qdrant's cosine ranking.
+GENERATED_PENALTY = 0.15  # subtract from generated code hits
+ENTRYPOINT_BOOST = 0.05  # add to framework entrypoint files
+
+# Path patterns that match Angular / Node framework entrypoints worth surfacing
+# even when symbol-level similarity is lower than other hits.
+_ENTRYPOINT_BASENAMES = frozenset(
+    {
+        "app.config.ts",
+        "app.routes.ts",
+        "app-routing.module.ts",
+        "main.ts",
+        "main.server.ts",
+        "app.module.ts",
+        "app.component.ts",
+        "index.ts",
+        "providers.ts",
+    }
+)
+_ENTRYPOINT_SUFFIXES = (".module.ts", ".routing.ts", ".routes.ts", ".config.ts")
+
+
+def _is_entrypoint(path: str | None) -> bool:
+    if not path:
+        return False
+    name = Path(path).name.lower()
+    if name in _ENTRYPOINT_BASENAMES:
+        return True
+    return any(name.endswith(suf) for suf in _ENTRYPOINT_SUFFIXES)
+
+
+def _normalize_prompt(text: str) -> str:
+    """Lowercase + collapse whitespace for cheap near-duplicate detection."""
+    return re.sub(r"\s+", " ", text.strip().lower())[:160]
+
+
+@dataclass
+class ContextPack:
+    """Orientation payload for a natural-language query.
+
+    Topology questions (who calls X, who imports Y, …) deliberately do
+    **not** live here — they have dedicated MCP tools so the agent can
+    issue precise graph queries instead of skimming a noisy neighbor
+    dump.
+
+    User claims (Graphiti-style ``(subject, predicate, object)`` facts
+    extracted from prior prompts) DO live here when the local
+    ``claims.db`` has rows relevant to the query — they're cheap to read
+    and answer "what has the user already told me about X?" without
+    re-reading every past prompt.
+    """
+
+    query: str
+    code_hits: list[VectorHit] = field(default_factory=list)
+    episode_hits: list[VectorHit] = field(default_factory=list)
+    episodes: list[Episode] = field(default_factory=list)
+    claims: list[ClaimRecord] = field(default_factory=list)
+
+    def render(self) -> str:
+        lines = [f"# Query\n{self.query}\n"]
+        if self.code_hits:
+            lines.append("## Code")
+            for h in self.code_hits:
+                p = h.payload
+                lines.append(
+                    f"- {p.get('path')}:{p.get('start')}-{p.get('end')} "
+                    f"[{p.get('kind')} {p.get('name')}] score={h.score:.3f}"
+                )
+        if self.episode_hits:
+            lines.append("\n## Episodes")
+            for ep in self.episodes:
+                lines.append(
+                    f"- {ep.id} verdict={ep.verdict} :: {ep.prompt[:120]}"
+                )
+        if self.claims:
+            lines.append("\n## User claims")
+            for c in self.claims:
+                neg = "" if c.polarity else " (NEGATED)"
+                lines.append(
+                    f"- {c.subject} {c.predicate} {c.object}{neg} "
+                    f"(conf={c.confidence:.2f})"
+                )
+        lines.append(
+            "\n_For topology (callers/callees/importers/dependencies/definitions) "
+            "use the dedicated codememory_* tools._"
+        )
+        return "\n".join(lines)
+
+    def to_dict(self) -> dict[str, Any]:
+        """Machine-readable representation for plugin / tool consumers."""
+        return {
+            "query": self.query,
+            "code": [
+                {
+                    "path": h.payload.get("path"),
+                    "start": h.payload.get("start"),
+                    "end": h.payload.get("end"),
+                    "kind": h.payload.get("kind"),
+                    "name": h.payload.get("name"),
+                    "score": h.score,
+                }
+                for h in self.code_hits
+            ],
+            "episodes": [
+                {
+                    "id": ep.id,
+                    "verdict": ep.verdict,
+                    "prompt": ep.prompt[:240],
+                    "score": next(
+                        (h.score for h in self.episode_hits if h.id == ep.id),
+                        None,
+                    ),
+                }
+                for ep in self.episodes
+            ],
+            "claims": [
+                {
+                    "subject": c.subject,
+                    "predicate": c.predicate,
+                    "object": c.object,
+                    "polarity": c.polarity,
+                    "confidence": c.confidence,
+                    "valid_at": c.valid_at,
+                    "head_sha": c.head_sha,
+                }
+                for c in self.claims
+            ],
+        }
+
+
+class Retriever:
+    """Vector + episode retrieval. Topology lives in dedicated MCP tools."""
+
+    def __init__(
+        self,
+        project: str | None = None,
+        embedder: M3Embedder | None = None,
+        vector: QdrantStore | None = None,
+        episodic: EpisodicStore | None = None,
+        claims_indexer: ClaimsIndexer | None = None,
+    ) -> None:
+        self.slug = project or detect_project_slug()
+        self.cfg: Config = CONFIG.for_project(self.slug)
+        self.embedder = embedder or get_embedder()
+        self.vector = vector or QdrantStore()
+        self.episodic = episodic or EpisodicStore(path=self.cfg.episodic_db)
+        # Injected for tests; lazily built on first claim retrieval in
+        # prod so projects without ``claims.db`` pay zero cost.
+        self._claims_indexer = claims_indexer
+        # Mirror writer fallback in mcp_server / cli: when env var is
+        # unset, persist alongside the other per-project SQLite stores.
+        env_metrics = os.environ.get(ENV_METRICS)
+        self._metrics_path = Path(env_metrics) if env_metrics else CONFIG.data_dir / "metrics.db"
+        self._metrics: MetricsStore | None = None
+
+    def retrieve(
+        self,
+        query: str,
+        top_k_code: int = 8,
+        top_k_eps: int = 5,
+        top_k_claims: int = 5,
+        include_idle_episodes: bool = False,
+    ) -> ContextPack:
+        t0 = time.time()
+
+        qvec = self.embedder.embed_one(query)
+        t_embed = time.time()
+
+        # Fetch 2x candidates so re-rank has room to lift entrypoints
+        # and demote generated code without losing depth. Mode is
+        # selected per ``CODEMEMORY_HYBRID``; default ``dense`` reflects
+        # the benchmark winner — see docs/BENCHMARK.md.
+        raw_code = self.vector.search(
+            self.cfg.qdrant_code,
+            qvec,
+            top_k=top_k_code * 2,
+            mode=_retrieval_mode(),
+        )
+        t_code = time.time()
+        code_hits = _rerank_code(raw_code)[:top_k_code]
+
+        raw_eps = self.vector.search(
+            self.cfg.qdrant_episodes, qvec, top_k=top_k_eps * 3
+        )
+        t_eps = time.time()
+        episodes = self.episodic.by_ids([h.id for h in raw_eps])
+        ep_hits, episodes = _filter_episodes(
+            query,
+            raw_eps,
+            episodes,
+            limit=top_k_eps,
+            include_idle=include_idle_episodes,
+        )
+
+        claims = self._retrieve_claims(query, qvec, limit=top_k_claims)
+        t_claims = time.time()
+
+        total = time.time() - t0
+
+        # Timing log
+        log.debug(
+            "retrieve query=%r embed=%.0fms code=%.0fms eps=%.0fms claims=%.0fms total=%.0fms "
+            "code_hits=%d eps_hits=%d claims_hits=%d",
+            query,
+            (t_embed - t0) * 1000,
+            (t_code - t_embed) * 1000,
+            (t_eps - t_code) * 1000,
+            (t_claims - t_eps) * 1000,
+            total * 1000,
+            len(code_hits),
+            len(ep_hits),
+            len(claims),
+        )
+
+        # Warn on slow stages
+        if (t_embed - t0) > 5.0:
+            log.warning(
+                "retrieve: embed took %.1fs for query=%r", t_embed - t0, query
+            )
+        if (t_code - t_embed) > 2.0:
+            log.warning(
+                "retrieve: code search took %.1fs for query=%r",
+                t_code - t_embed,
+                query,
+            )
+
+        # Record metrics (non-fatal)
+        if self._metrics_path and self._metrics_path.suffix == ".db":
+            try:
+                if self._metrics is None:
+                    self._metrics = MetricsStore(self._metrics_path)
+                self._metrics.record_retrieve(
+                    RetrieveTiming(
+                        query=query,
+                        embed_ms=(t_embed - t0) * 1000,
+                        code_search_ms=(t_code - t_embed) * 1000,
+                        eps_search_ms=(t_eps - t_code) * 1000,
+                        claims_ms=(t_claims - t_eps) * 1000,
+                        total_ms=total * 1000,
+                        code_hit_count=len(code_hits),
+                        eps_hit_count=len(ep_hits),
+                        claims_hit_count=len(claims),
+                    )
+                )
+            except Exception:
+                pass  # metrics never fail retrieval
+
+        return ContextPack(
+            query=query,
+            code_hits=code_hits,
+            episode_hits=ep_hits,
+            episodes=episodes,
+            claims=claims,
+        )
+
+    def _retrieve_claims(
+        self,
+        query: str,
+        query_vec: HybridVec,
+        limit: int,
+    ) -> list[ClaimRecord]:
+        """Best-effort claim recall.
+
+        Two-stage strategy:
+
+        1. **Semantic.** If the per-project Qdrant claim collection
+           exists (or can be backfilled from SQLite), search it and
+           hydrate the top-k via ``store.by_ids``. Reranked by cosine
+           score × confidence × recency decay.
+        2. **Token-overlap fallback.** When no claims collection is
+           reachable (Qdrant down, embedder unavailable, or the
+           project never opted into claim extraction), fall back to
+           the lexical ``_rank_claims`` heuristic.
+
+        Returns empty when the per-project ``claims.db`` doesn't exist
+        (the common case for projects that never opted into extraction)
+        so this is free for non-claim users.
+        """
+        if limit <= 0:
+            return []
+        path = self.cfg.claims_db
+        if not path.exists():
+            return []
+
+        indexer = self._ensure_claims_indexer()
+        if indexer is not None:
+            ranked = _semantic_claim_search(indexer, query_vec, limit)
+            if ranked is not None:
+                return ranked[:limit]
+
+        # Fallback path: SQLite-only token-overlap. Still useful when
+        # the embedder is offline (CI without Ollama, dev box without
+        # GPU, etc.) — keeps the orientation payload non-empty.
+        try:
+            store = ClaimsStore(path=path)
+        except Exception:  # noqa: BLE001
+            return []
+        try:
+            rows = store.current()
+        finally:
+            store.close()
+        if not rows:
+            return []
+        return _rank_claims(query, rows)[:limit]
+
+    def _ensure_claims_indexer(self) -> ClaimsIndexer | None:
+        """Lazy-init the indexer. Returns ``None`` if construction fails.
+
+        Failure modes are silent on purpose: a broken Qdrant or
+        embedder must not take down the whole retrieve call — the
+        fallback path will still return token-overlap claims.
+        """
+        if self._claims_indexer is not None:
+            return self._claims_indexer
+        try:
+            self._claims_indexer = make_claims_indexer(
+                project=self.slug,
+                cfg=self.cfg,
+                embedder=self.embedder,
+                vector=self.vector,
+            )
+        except Exception:  # noqa: BLE001
+            self._claims_indexer = None
+            return None
+        return self._claims_indexer
+
+
+def _rerank_code(hits: list[VectorHit]) -> list[VectorHit]:
+    """Apply generated-code penalty + entrypoint boost; resort by score.
+
+    Returns new ``VectorHit`` instances so caller doesn't see mutated cosine
+    scores from Qdrant.
+    """
+    adjusted: list[VectorHit] = []
+    for h in hits:
+        score = h.score
+        path = h.payload.get("path")
+        if h.payload.get("generated"):
+            score -= GENERATED_PENALTY
+        if _is_entrypoint(path):
+            score += ENTRYPOINT_BOOST
+        adjusted.append(VectorHit(id=h.id, score=score, payload=h.payload))
+    adjusted.sort(key=lambda h: h.score, reverse=True)
+    return adjusted
+
+
+def _semantic_claim_search(
+    indexer: ClaimsIndexer,
+    query_vec: HybridVec,
+    limit: int,
+) -> list[ClaimRecord] | None:
+    """Run a Qdrant search over claim points and hydrate rows.
+
+    Returns ``None`` when the indexer's collection is missing AND
+    backfill produced no rows — the caller falls back to lexical
+    ranking. Returns ``[]`` (not ``None``) when the collection exists
+    but yielded no semantic matches: the lexical path would just
+    re-scan the same rows for nothing, so we short-circuit.
+    """
+    try:
+        embedded = indexer.ensure_backfilled()
+    except Exception:  # noqa: BLE001
+        return None
+    if embedded == 0 and indexer.store.count() == 0:
+        return None
+
+    hits = indexer.search(query_vec, top_k=limit * 3)
+    if not hits:
+        # Backfill ran (or skipped because already in sync) but no
+        # semantic neighbors — surface nothing rather than recompute
+        # the same answer via token overlap.
+        return []
+
+    claims = indexer.store.by_ids([h.id for h in hits])
+    by_id = {c.id: c for c in claims}
+    ranked: list[tuple[float, ClaimRecord]] = []
+    now = time.time()
+    for h in hits:
+        claim = by_id.get(h.id)
+        if claim is None:
+            continue
+        age_s = max(0.0, now - claim.valid_at)
+        decay = math.exp(-age_s / _CLAIM_RECENCY_HALF_LIFE_S)
+        score = h.score * claim.confidence * (0.5 + 0.5 * decay)
+        ranked.append((score, claim))
+    ranked.sort(key=lambda pair: pair[0], reverse=True)
+    return [c for _, c in ranked]
+
+
+_CLAIM_TOKEN_RE = re.compile(r"[A-Za-z][A-Za-z0-9_-]+")
+# Recency half-life: a claim asserted 30 days ago weighs half as much
+# as one asserted today, all else equal. Long enough that durable
+# preferences ("we use Postgres") survive across sessions; short enough
+# that abandoned-then-reasserted facts surface fresh.
+_CLAIM_RECENCY_HALF_LIFE_S = 30 * 24 * 3600
+
+
+def _claim_tokens(text: str) -> set[str]:
+    return {m.group(0).lower() for m in _CLAIM_TOKEN_RE.finditer(text or "")}
+
+
+def _rank_claims(query: str, claims: list[ClaimRecord]) -> list[ClaimRecord]:
+    """Score claims by query-token overlap + recency + confidence.
+
+    Cheap token-bag scoring is good enough for the current scale (tens
+    to hundreds of claims per project). Once volumes justify it we'll
+    swap in vector relevance via a dedicated Qdrant collection (see the
+    entity-resolution work).
+    """
+    q_tokens = _claim_tokens(query)
+    if not q_tokens:
+        # No tokens to match — fall back to recency * confidence.
+        return sorted(
+            claims,
+            key=lambda c: (c.confidence, c.valid_at),
+            reverse=True,
+        )
+
+    now = time.time()
+    scored: list[tuple[float, ClaimRecord]] = []
+    for c in claims:
+        bag = _claim_tokens(c.subject) | _claim_tokens(c.object)
+        # evidence_span is verbatim user text — strong signal when it
+        # mentions the query terms, so we count it in the bag too.
+        bag |= _claim_tokens(c.evidence_span)
+        overlap = len(bag & q_tokens)
+        if overlap == 0:
+            continue
+        age_s = max(0.0, now - c.valid_at)
+        decay = math.exp(-age_s / _CLAIM_RECENCY_HALF_LIFE_S)
+        score = overlap * c.confidence * (0.5 + 0.5 * decay)
+        scored.append((score, c))
+    scored.sort(key=lambda pair: pair[0], reverse=True)
+    return [c for _, c in scored]
+
+
+def _filter_episodes(
+    query: str,
+    hits: list[VectorHit],
+    episodes: list[Episode],
+    *,
+    limit: int,
+    include_idle: bool,
+) -> tuple[list[VectorHit], list[Episode]]:
+    """Drop idle verdicts (opt-in) and dedupe near-identical prompts.
+
+    Episodes whose normalized prompt prefix matches the current query or
+    another already-kept episode are suppressed. This eliminates the
+    "10 copies of my own prior question" noise without needing a second
+    embedding round-trip.
+    """
+    by_id = {ep.id: ep for ep in episodes}
+    query_key = _normalize_prompt(query)
+    kept_hits: list[VectorHit] = []
+    kept_eps: list[Episode] = []
+    seen_keys: set[str] = {query_key}
+    for h in hits:
+        ep = by_id.get(h.id)
+        if ep is None:
+            continue
+        if not include_idle and (ep.verdict or "").lower() == "idle":
+            continue
+        key = _normalize_prompt(ep.prompt or "")
+        if key in seen_keys:
+            continue
+        seen_keys.add(key)
+        kept_hits.append(h)
+        kept_eps.append(ep)
+        if len(kept_hits) >= limit:
+            break
+    return kept_hits, kept_eps

code_memory/resilience.py

@@ -0,0 +1,73 @@
+import time
+import httpx
+
+def is_retryable(exc: Exception) -> bool:
+    """True for transient failures that retry can fix."""
+    if isinstance(exc, httpx.ConnectError):
+        return True
+    if isinstance(exc, httpx.TimeoutException):
+        return True
+    if isinstance(exc, httpx.HTTPStatusError):
+        return exc.response.status_code >= 500
+    return False
+
+def with_retry(fn, *, max_retries=3, backoff_s=1.0, on_retry=None):
+    """Call fn(), retry on transient httpx errors with exponential backoff.
+    
+    After max_retries attempts, re-raises the last exception.
+    on_retry(attempt, exc) is called before each retry for logging.
+    """
+    for attempt in range(max_retries + 1):
+        try:
+            return fn()
+        except (httpx.ConnectError, httpx.TimeoutException, httpx.HTTPStatusError) as exc:
+            if attempt == max_retries:
+                raise
+            if not is_retryable(exc):
+                raise
+            if on_retry:
+                on_retry(attempt + 1, exc)
+            delay = backoff_s * (2 ** attempt)
+            time.sleep(delay)
+    raise RuntimeError("unreachable")
+
+class CircuitBreaker:
+    """Opens after `threshold` consecutive failures; stays open for `cooldown_s`.
+    
+    While open, raises CircuitBreakerOpenError immediately without calling fn.
+    On first success after cooldown, transitions to half-open; on next success, closes.
+    """
+    
+    def __init__(self, name="default", threshold=5, cooldown_s=30.0):
+        self.name = name
+        self.threshold = threshold
+        self.cooldown_s = cooldown_s
+        self._failures = 0
+        self._last_failure = 0.0
+        self._state = "closed"  # closed | open | half_open
+    
+    @property
+    def state(self) -> str:
+        if self._state == "open" and time.time() - self._last_failure > self.cooldown_s:
+            self._state = "half_open"
+        return self._state
+    
+    def call(self, fn, *args, **kwargs):
+        if self._state == "open" or (self._state == "open" and time.time() - self._last_failure <= self.cooldown_s):
+            raise CircuitBreakerOpenError(self.name, self._failures)
+        try:
+            result = fn(*args, **kwargs)
+            if self._state == "half_open":
+                self._state = "closed"
+            self._failures = 0
+            return result
+        except Exception as exc:
+            self._failures += 1
+            self._last_failure = time.time()
+            if self._failures >= self.threshold:
+                self._state = "open"
+            raise
+
+class CircuitBreakerOpenError(Exception):
+    def __init__(self, name, failures):
+        super().__init__(f"Circuit breaker '{name}' open after {failures} failures")

code_memory/sync/__init__.py

@@ -0,0 +1,20 @@
+"""Team-shared code-memory: snapshot, sync, hooks, autostart, watcher."""
+
+from .snapshot import (
+    Snapshot,
+    SnapshotManifest,
+    apply_snapshot,
+    build_snapshot,
+    verify_snapshot,
+)
+from .sync import SyncResult, sync_repo
+
+__all__ = [
+    "Snapshot",
+    "SnapshotManifest",
+    "SyncResult",
+    "apply_snapshot",
+    "build_snapshot",
+    "sync_repo",
+    "verify_snapshot",
+]

code_memory/sync/autostart/__init__.py

@@ -0,0 +1,42 @@
+"""Cross-platform autostart registration for the code-memory watcher.
+
+Adapters write a user-level service unit that runs ``code-memory watch <repo>``
+at user logon. No root/admin required.
+
+  - macOS   -> launchd LaunchAgent (~/Library/LaunchAgents/*.plist)
+  - Linux   -> systemd --user unit (~/.config/systemd/user/*.service)
+  - Windows -> Scheduled Task at logon (schtasks /SC ONLOGON /RL LIMITED)
+"""
+
+from __future__ import annotations
+
+import platform
+from pathlib import Path
+from typing import Protocol
+
+from .base import AutostartStatus, ensure_autostart, get_adapter
+from .launchd import LaunchdAdapter
+from .schtasks import SchtasksAdapter
+from .systemd import SystemdUserAdapter
+
+__all__ = [
+    "AutostartStatus",
+    "LaunchdAdapter",
+    "SchtasksAdapter",
+    "SystemdUserAdapter",
+    "ensure_autostart",
+    "get_adapter",
+    "Adapter",
+]
+
+
+class Adapter(Protocol):
+    def install(self, repo: Path) -> AutostartStatus: ...
+    def uninstall(self, repo: Path) -> AutostartStatus: ...
+    def status(self, repo: Path) -> AutostartStatus: ...
+    def start(self, repo: Path) -> AutostartStatus: ...
+
+
+# convenience re-export
+def current_platform() -> str:
+    return platform.system()