sup-mem 0.7.0__py3-none-any.whl

sup_mem/__init__.py

@@ -0,0 +1,22 @@
+"""sup-mem: a self-hosted, pluggable global memory layer for Claude.
+
+Two front-doors (an automatic hook + explicit MCP tools) over one pluggable backend (I1).
+
+Keep this module import-light: the per-prompt hook imports from the package on the hot path
+(I2), so we import only stdlib-backed modules here (``config``/``models``) and NEVER touch
+``backends`` or ``embedding`` at import time.
+"""
+
+from __future__ import annotations
+
+from importlib.metadata import PackageNotFoundError, version
+
+from sup_mem.config import Config, load_config
+from sup_mem.models import Hit, MemoryRecord
+
+try:
+    __version__ = version("sup-mem")
+except PackageNotFoundError:  # running from a source tree without an install
+    __version__ = "0.0.0"
+
+__all__ = ["Config", "Hit", "MemoryRecord", "__version__", "load_config"]

sup_mem/archival.py

@@ -0,0 +1,165 @@
+"""Archival policy engine (docs/PHASE9-ARCHIVAL.md) — the three regimes of A2.
+
+1. Steady state: structurally-outdated versions (superseded past the window) and stale
+   quarantined memories move to the cold tier — evidence, never age (A1).
+2. Main pressure: over ``archival.main_max_mb``, decay candidates move most-useless-first
+   (ledger-ranked) until the cap is met — or candidates run out, in which case we STOP and
+   say so rather than archive evidently-useful memories.
+3. Archive pressure: over ``archival.archive_max_mb``, the oldest archived rows are deleted
+   forever, FIFO, chain-audited.
+
+The mechanics live on the sqlite backend; this module only decides *what* and *in which
+order*. Backends without archival support are reported as such, never guessed at.
+"""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime, timedelta
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from sup_mem.config import Config
+
+_MB = 1024 * 1024
+
+
+def _cutoff(days: int) -> str:
+    return (datetime.now(UTC) - timedelta(days=days)).isoformat()
+
+
+def _stale_quarantined_ids(config: Config) -> list[str]:
+    """Quarantined per the L3 rule AND unreferenced past the quarantine window."""
+    if not config.ledger_db_path.exists():
+        return []
+    from sup_mem.ledger import Ledger
+
+    threshold = _cutoff(config.archival.quarantined_after_days)
+    out: list[str] = []
+    with Ledger(config.ledger_db_path) as ledger:
+        for s in ledger.all_stats():
+            quarantined = (
+                s["contradicted"] >= config.ledger.quarantine_contradictions
+                and s["contradicted"] > s["referenced"]
+            )
+            stale = not s["last_referenced"] or s["last_referenced"] < threshold
+            if quarantined and stale:
+                out.append(str(s["memory_id"]))
+    return out
+
+
+def _rank_by_uselessness(candidates: list[dict[str, Any]], config: Config) -> list[dict[str, Any]]:
+    """Most-useless first (A1): evidenced-useless → no-evidence → (useful excluded upstream)."""
+    stats: dict[str, dict[str, Any]] = {}
+    if config.ledger_db_path.exists():
+        from sup_mem.ledger import Ledger
+
+        with Ledger(config.ledger_db_path) as ledger:
+            stats = {s["memory_id"]: s for s in ledger.all_stats()}
+
+    recent_ref_cutoff = _cutoff(config.archival.decay_min_age_days)
+    eligible: list[dict[str, Any]] = []
+    for cand in candidates:
+        s = stats.get(cand["id"], {})
+        last_ref = str(s.get("last_referenced", ""))
+        if last_ref and last_ref >= recent_ref_cutoff:
+            continue  # referenced recently → never pressure-archived (A1)
+        cand = dict(cand)
+        cand["_referenced"] = int(s.get("referenced", 0))
+        cand["_ignored"] = int(s.get("ignored", 0))
+        cand["_injected"] = int(s.get("injected", 0))
+        eligible.append(cand)
+
+    def key(c: dict[str, Any]) -> tuple[int, int, int, int, str]:
+        evidenced_useless = c["_injected"] > 0 and c["_referenced"] == 0
+        return (
+            0 if evidenced_useless else 1,  # proven-useless first, no-evidence after
+            c["_referenced"],  # fewer references = more useless
+            -c["_ignored"],  # more ignored-despite-chances first
+            -c["_injected"],
+            c["recorded_at"],  # oldest first as the final tiebreak
+        )
+
+    return sorted(eligible, key=key)
+
+
+def run_archival(config: Config, *, dry_run: bool = False) -> dict[str, Any]:
+    from sup_mem.backends import get_backend
+    from sup_mem.backends.sqlite_fts import SqliteFtsBackend
+
+    backend = get_backend(config)
+    if not isinstance(backend, SqliteFtsBackend):
+        backend.close()
+        return {"supported": False, "note": "archival requires the sqlite_fts backend"}
+
+    report: dict[str, Any] = {
+        "supported": True,
+        "steady": [],
+        "pressure": [],
+        "purged": [],
+        "note": "",
+    }
+    try:
+        report["sizes_before"] = backend.db_sizes()
+
+        # Regime 1 — steady state: structural + proven-harmful tiers.
+        steady_ids = list(
+            dict.fromkeys(
+                backend.superseded_before(_cutoff(config.archival.superseded_after_days))
+                + _stale_quarantined_ids(config)
+            )
+        )
+        if dry_run:
+            report["steady"] = steady_ids
+        elif steady_ids:
+            report["steady"] = backend.archive_versions(steady_ids)
+
+        # Regime 2 — main pressure: most-useless-first until under the cap.
+        main_cap = int(config.archival.main_max_mb * _MB)
+        if main_cap > 0:
+            size = backend.db_sizes()["main"] if not dry_run else report["sizes_before"]["main"]
+            if size > main_cap:
+                ranked = _rank_by_uselessness(
+                    backend.live_candidates(
+                        _cutoff(config.archival.decay_min_age_days), config.archival.keep_tag
+                    ),
+                    config,
+                )
+                if dry_run:
+                    report["pressure"] = [c["id"] for c in ranked]
+                    report["note"] = (
+                        f"main tier over cap ({size / _MB:.1f} > "
+                        f"{config.archival.main_max_mb} MB); would archive up to "
+                        f"{len(ranked)} candidates, most-useless first"
+                    )
+                else:
+                    while ranked and backend.db_sizes()["main"] > main_cap:
+                        batch, ranked = ranked[:25], ranked[25:]
+                        report["pressure"].extend(
+                            backend.archive_versions([c["id"] for c in batch])
+                        )
+                        backend.compact()
+                    if backend.db_sizes()["main"] > main_cap:
+                        report["note"] = (
+                            "main tier still over cap after archiving every eligible "
+                            "candidate — refusing to archive evidently-useful memories; "
+                            "raise archival.main_max_mb"
+                        )
+
+        # Regime 3 — archive pressure: FIFO permanent deletion (chain-audited).
+        archive_cap = int(config.archival.archive_max_mb * _MB)
+        if archive_cap > 0 and not dry_run:
+            report["purged"] = backend.purge_archive_fifo(archive_cap)
+        elif archive_cap > 0 and dry_run:
+            over = backend.db_sizes()["archive"] - archive_cap
+            if over > 0:
+                report["note"] += (
+                    f" archive tier over cap by {over / _MB:.1f} MB; oldest archived rows "
+                    "would be DELETED FOREVER (FIFO)"
+                )
+
+        if not dry_run:
+            backend.compact()
+        report["sizes_after"] = backend.db_sizes()
+        return report
+    finally:
+        backend.close()

sup_mem/backends/__init__.py

@@ -0,0 +1,37 @@
+"""Pluggable storage backends behind one interface (I6).
+
+``get_backend`` is the single construction point used by the hook, the MCP server and the CLI.
+Backend modules are imported LAZILY here so that:
+  * the zero-optional-deps default install never imports ``qdrant_client`` / ``fastembed`` (I8),
+  * the hook's hot path only pays for the backend it actually uses (I2).
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from sup_mem.backends.base import MemoryBackend
+    from sup_mem.config import Config
+
+
+def get_backend(config: Config) -> MemoryBackend:
+    """Construct the configured backend. Raises ``ValueError`` for an unknown name."""
+    name = config.backend
+    if name == "sqlite_fts":
+        from sup_mem.backends.sqlite_fts import SqliteFtsBackend
+
+        return SqliteFtsBackend(config)
+    if name == "qdrant":
+        from sup_mem.backends.qdrant import QdrantBackend
+
+        return QdrantBackend(config)
+    if name == "pgvector":
+        # Documented v1 stub (§14): the interface is reserved; not implemented.
+        raise NotImplementedError(
+            "The pgvector backend is a documented v1 stub (§14). Use 'sqlite_fts' or 'qdrant'."
+        )
+    raise ValueError(f"Unknown backend {name!r}. Expected one of: sqlite_fts, qdrant, pgvector.")
+
+
+__all__ = ["get_backend"]

sup_mem/backends/base.py

@@ -0,0 +1,78 @@
+"""The one interface every backend implements (HANDOVER §6.1, I6).
+
+Everything above this line — the hook, the MCP server, the manifest — depends ONLY on this
+abstract class. Adding a new backend must not require touching any of them.
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from collections.abc import Callable
+from typing import Any
+
+from sup_mem.models import Hit, Metadata
+
+# Called as progress(done, total) during a reindex so the CLI can render a bar.
+ProgressCallback = Callable[[int, int], None]
+
+
+class MemoryBackend(ABC):
+    """Shared storage for both front-doors (the hook and the MCP tools), I1."""
+
+    @abstractmethod
+    def store(self, text: str, metadata: Metadata | None = None) -> str:
+        """Persist a memory and return its id. Idempotent on ``(text, source)`` where the
+        backend can manage it (``source`` is read from ``metadata['source']``)."""
+
+    @abstractmethod
+    def search(self, query: str, k: int, threshold: float, as_of: str | None = None) -> list[Hit]:
+        """Return up to ``k`` hits with ``score >= threshold``, best first.
+
+        ``score`` MUST be normalized to 0..1 so ``threshold`` is portable across backends
+        (§6.1): BM25 backends squash their rank; cosine backends map naturally.
+
+        ``as_of`` (ISO timestamp) asks for transaction-time-of-record results — "what was
+        live in the store at that instant" (PHASE8 T2). Backends without version history
+        MUST raise ``ValueError`` for a non-None ``as_of`` rather than silently returning
+        current results (T6).
+        """
+
+    @abstractmethod
+    def manifest(self, max_topics: int) -> list[str]:
+        """Return a compact topic index, never longer than ``max_topics`` (§6.7)."""
+
+    @abstractmethod
+    def health(self) -> dict[str, Any]:
+        """Liveness + config summary.
+
+        Keys: ``backend`` (str), ``count`` (int), ``embedding`` (``{provider, model, dim}``
+        for vector backends, else ``None``), plus any backend-specific extras.
+        """
+
+    @abstractmethod
+    def reindex(self, progress: ProgressCallback | None = None) -> None:
+        """Re-embed / rebuild the store. No-op for lexical backends; required for vector (I7)."""
+
+    @abstractmethod
+    def fetch(self, memory_ids: list[str]) -> dict[str, str]:
+        """Return ``{id: text}`` for the ids that exist. Used by the outcome ledger's
+        attribution (docs/PHASE6-LOOP.md, L5); missing ids are simply omitted."""
+
+    @property
+    def hook_safe(self) -> bool:
+        """Whether the per-prompt hook may call ``search()`` without loading a model (I2).
+
+        Lexical backends and remote-embedder vector backends are safe; an in-process embedder
+        (fastembed) is not, so the hook skips Tier-2 for it. Defaults to True.
+        """
+        return True
+
+    # --- Lifecycle (optional to override) -------------------------------------------------
+    def close(self) -> None:  # noqa: B027
+        """Release resources (DB handles, clients). Optional override; default no-op."""
+
+    def __enter__(self) -> MemoryBackend:
+        return self
+
+    def __exit__(self, *exc: object) -> None:
+        self.close()

sup_mem/backends/qdrant.py

@@ -0,0 +1,317 @@
+"""Opt-in vector backend: Qdrant + a pluggable embedder (HANDOVER §6.3, I7).
+
+Embedding happens via the configured provider (§6.4). The per-prompt hook only calls this
+backend when ``hook_safe`` is true (a remote/warm embedder), so the short-lived hook never
+loads a model (I2); fastembed is used from the warm MCP server or batch paths.
+
+I7 (model-consistency) is a hard contract: the ``(provider, model, dim)`` that wrote the store
+is persisted as a reserved meta point INSIDE Qdrant, so it travels with the vectors. Any
+attempt to read/write with a different model raises with a `reindex` remediation.
+"""
+
+from __future__ import annotations
+
+import re
+import uuid
+from datetime import UTC, datetime
+from typing import TYPE_CHECKING, Any
+
+from sup_mem.backends.base import MemoryBackend, ProgressCallback
+from sup_mem.embedding import get_embedder
+from sup_mem.embedding.base import (
+    Embedder,
+    EmbeddingError,
+    EmbeddingMeta,
+    provider_is_hook_safe,
+)
+from sup_mem.models import Hit, Metadata
+
+if TYPE_CHECKING:
+    from sup_mem.config import Config
+
+_NAMESPACE = uuid.UUID("6f9619ff-8b86-d011-b42d-00cf4fc964ff")  # stable id namespace
+_META_ID = str(uuid.uuid5(_NAMESPACE, "__sup_mem_meta__"))
+_META_FLAG = "__cm_meta__"
+_WORD_RE = re.compile(r"\w", re.UNICODE)
+_REINDEX_HINT = "Run `sup-mem reindex` to re-embed the store with the current model."
+
+
+def _now_iso() -> str:
+    return datetime.now(UTC).isoformat()
+
+
+def _point_id(text: str, source: str) -> str:
+    return str(uuid.uuid5(_NAMESPACE, f"{source}\x00{text}"))
+
+
+def _valid_point_id(value: str) -> bool:
+    if value.isdigit():
+        return True
+    try:
+        uuid.UUID(value)
+    except ValueError:
+        return False
+    return True
+
+
+class QdrantBackend(MemoryBackend):
+    def __init__(self, config: Config) -> None:
+        from qdrant_client import QdrantClient
+        from qdrant_client import models as qm
+
+        self._config = config
+        self._qm = qm
+        self._collection = config.qdrant.collection
+        # Our API surface (collections/upsert/query_points/scroll/count) is stable across
+        # server versions, so skip the client's strict minor-version compatibility warning.
+        self._client = QdrantClient(url=config.qdrant.url, check_compatibility=False)
+        self._embedder_cache: Embedder | None = None
+        self._stored_meta: EmbeddingMeta | None = None
+
+    # -- embedder + I7 --------------------------------------------------------------------
+    def _embedder(self) -> Embedder:
+        if self._embedder_cache is None:
+            self._embedder_cache = get_embedder(self._config)
+        return self._embedder_cache
+
+    @property
+    def hook_safe(self) -> bool:
+        # Decided from config alone — never constructs the embedder (I2).
+        return provider_is_hook_safe(self._config.embedding.provider)
+
+    def _read_stored_meta(self) -> EmbeddingMeta | None:
+        if self._stored_meta is not None:
+            return self._stored_meta
+        points = self._client.retrieve(self._collection, ids=[_META_ID], with_payload=True)
+        if not points:
+            return None
+        payload = points[0].payload or {}
+        self._stored_meta = EmbeddingMeta.from_dict(payload)
+        return self._stored_meta
+
+    def _configured_identity(self) -> tuple[str, str]:
+        from sup_mem.embedding import providers
+
+        provider = self._config.embedding.provider
+        spec = providers.SPEC_BY_NAME.get(provider)
+        model = self._config.embedding.model or (spec.default_model if spec else "")
+        return provider, model
+
+    def _verify_consistency(self) -> None:
+        """Raise if the configured model differs from what wrote the store (I7)."""
+        stored = self._read_stored_meta()
+        if stored is None:
+            return
+        provider, model = self._configured_identity()
+        if (stored.provider, stored.model) != (provider, model):
+            raise EmbeddingError(
+                f"Embedding-model mismatch (I7): store was written with "
+                f"'{stored.provider}/{stored.model}' but config selects '{provider}/{model}'. "
+                f"Vectors from different models are not comparable. {_REINDEX_HINT}"
+            )
+
+    def check_consistency(self) -> None:
+        """Public I7 check for `doctor` — no-op if the store is empty/uninitialized."""
+        if not self._client.collection_exists(self._collection):
+            return
+        self._verify_consistency()
+
+    # -- collection lifecycle -------------------------------------------------------------
+    def _create_collection(self, dim: int) -> None:
+        qm = self._qm
+        hnsw = self._config.qdrant.hnsw
+        quant = None
+        if self._config.qdrant.quantization:
+            quant = qm.ScalarQuantization(
+                scalar=qm.ScalarQuantizationConfig(type=qm.ScalarType.INT8, always_ram=True)
+            )
+        self._client.create_collection(
+            collection_name=self._collection,
+            vectors_config=qm.VectorParams(size=dim, distance=qm.Distance.COSINE),
+            hnsw_config=qm.HnswConfigDiff(m=hnsw.m, ef_construct=hnsw.ef_construct),
+            quantization_config=quant,
+        )
+
+    def _write_meta(self, meta: EmbeddingMeta) -> None:
+        qm = self._qm
+        payload = {**meta.as_dict(), _META_FLAG: True}
+        self._client.upsert(
+            self._collection,
+            points=[qm.PointStruct(id=_META_ID, vector=[0.0] * meta.dim, payload=payload)],
+        )
+        self._stored_meta = meta
+
+    def _ensure_writable(self) -> Embedder:
+        embedder = self._embedder()
+        if not self._client.collection_exists(self._collection):
+            meta = embedder.meta
+            self._create_collection(meta.dim)
+            self._write_meta(meta)
+        else:
+            self._verify_consistency()
+        return embedder
+
+    def _not_meta(self) -> Any:
+        qm = self._qm
+        return qm.Filter(
+            must_not=[qm.FieldCondition(key=_META_FLAG, match=qm.MatchValue(value=True))]
+        )
+
+    def initialize(self) -> EmbeddingMeta:
+        """Create the collection (if needed) and record the embedding meta (I7); returns it.
+
+        Called by `sup-mem setup` — constructs the embedder to learn the vector `dim`.
+        """
+        return self._ensure_writable().meta
+
+    # -- writes ---------------------------------------------------------------------------
+    def store(self, text: str, metadata: Metadata | None = None) -> str:
+        text = text.strip()
+        meta = dict(metadata or {})
+        source = str(meta.get("source", ""))
+        embedder = self._ensure_writable()
+        vector = embedder.embed_query(text)
+        point_id = _point_id(text, source)
+        now = _now_iso()
+        payload = {
+            "text": text,
+            "metadata": meta,
+            "source": source,
+            "created_at": now,
+            "updated_at": now,
+        }
+        self._client.upsert(
+            self._collection,
+            points=[self._qm.PointStruct(id=point_id, vector=vector, payload=payload)],
+        )
+        return point_id
+
+    # -- reads ----------------------------------------------------------------------------
+    def search(self, query: str, k: int, threshold: float, as_of: str | None = None) -> list[Hit]:
+        if as_of is not None:
+            # No version history here — refusing beats silently answering with "now" (T6).
+            raise ValueError(
+                "bitemporal (--as-of) recall requires the sqlite_fts backend; "
+                "the qdrant backend stores only current versions."
+            )
+        if k <= 0 or not _WORD_RE.search(query):
+            return []
+        if not self._client.collection_exists(self._collection):
+            return []
+        embedder = self._embedder()
+        self._verify_consistency()
+        vector = embedder.embed_query(query)
+        result = self._client.query_points(
+            self._collection,
+            query=vector,
+            limit=k,
+            query_filter=self._not_meta(),
+            search_params=self._qm.SearchParams(hnsw_ef=self._config.qdrant.hnsw.ef),
+            with_payload=True,
+        )
+        hits: list[Hit] = []
+        for point in result.points:
+            score = max(0.0, min(1.0, float(point.score)))  # cosine → clamp to 0..1 (§6.1)
+            if score < threshold:
+                continue
+            payload = point.payload or {}
+            hits.append(
+                Hit(
+                    id=str(point.id),
+                    text=str(payload.get("text", "")),
+                    score=score,
+                    metadata=dict(payload.get("metadata", {})),
+                )
+            )
+        return hits
+
+    def fetch(self, memory_ids: list[str]) -> dict[str, str]:
+        # Qdrant point ids must be unsigned ints or UUIDs; anything else cannot exist in the
+        # collection, so it is "omitted" by definition rather than sent (the server 400s on it).
+        valid = [mid for mid in memory_ids if _valid_point_id(mid)]
+        if not valid or not self._client.collection_exists(self._collection):
+            return {}
+        points = self._client.retrieve(self._collection, ids=valid, with_payload=True)
+        return {
+            str(point.id): str((point.payload or {}).get("text", ""))
+            for point in points
+            if not (point.payload or {}).get(_META_FLAG)
+        }
+
+    # -- introspection --------------------------------------------------------------------
+    def _iter_payloads(self) -> Any:
+        offset = None
+        while True:
+            points, offset = self._client.scroll(
+                self._collection,
+                scroll_filter=self._not_meta(),
+                with_payload=True,
+                with_vectors=False,
+                limit=256,
+                offset=offset,
+            )
+            yield from points
+            if offset is None:
+                break
+
+    def manifest(self, max_topics: int) -> list[str]:
+        if max_topics <= 0 or not self._client.collection_exists(self._collection):
+            return []
+        from collections import Counter
+
+        counter: Counter[str] = Counter()
+        for point in self._iter_payloads():
+            meta = (point.payload or {}).get("metadata", {})
+            topic = meta.get("topic")
+            if isinstance(topic, str) and topic.strip():
+                counter[topic.strip()] += 1
+            tags = meta.get("tags", [])
+            if isinstance(tags, str):
+                tags = [t.strip() for t in tags.split(",")]
+            if isinstance(tags, list):
+                for tag in tags:
+                    if str(tag).strip():
+                        counter[str(tag).strip()] += 1
+        return [topic for topic, _ in counter.most_common(max_topics)]
+
+    def health(self) -> dict[str, Any]:
+        exists = self._client.collection_exists(self._collection)
+        count = 0
+        stored = None
+        if exists:
+            count = int(
+                self._client.count(
+                    self._collection, count_filter=self._not_meta(), exact=True
+                ).count
+            )
+            stored = self._read_stored_meta()
+        return {
+            "backend": "qdrant",
+            "count": count,
+            "embedding": stored.as_dict() if stored else None,
+            "revision": f"{count}:{stored.model if stored else ''}",
+            "url": self._config.qdrant.url,
+            "collection": self._collection,
+        }
+
+    def reindex(self, progress: ProgressCallback | None = None) -> None:
+        """Re-embed every memory with the CURRENT model and update stored meta (I7)."""
+        if not self._client.collection_exists(self._collection):
+            return
+        embedder = self._embedder()
+        meta = embedder.meta
+        points = list(self._iter_payloads())
+        total = len(points)
+        for done, point in enumerate(points, 1):
+            payload = point.payload or {}
+            vector = embedder.embed_query(str(payload.get("text", "")))
+            self._client.upsert(
+                self._collection,
+                points=[self._qm.PointStruct(id=point.id, vector=vector, payload=payload)],
+            )
+            if progress is not None:
+                progress(done, total)
+        self._write_meta(meta)  # store now matches the current model
+
+    def close(self) -> None:
+        self._client.close()