superlocalmemory 2.8.6 → 3.0.0

package/src/superlocalmemory/retrieval/agentic.py

@@ -0,0 +1,295 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under the MIT License - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""SuperLocalMemory V3 — 2-Round Sufficiency Verification (EverMemOS Pattern).
+
+Round 1: Standard retrieval → sufficiency check.
+Round 2 (if insufficient): LLM generates refined queries → merge → rerank.
+
+Design decisions:
+- 2 rounds MAX (3-round decomposition BROKE relational context in S16)
+- Trigger: max_score < 0.6 OR multi_hop query type
+- Skip agentic entirely for temporal queries (S15 lesson)
+- Mode A: heuristic alias expansion (no LLM)
+- Mode C: LLM sufficiency judgment with 3-way classification
+
+Part of Qualixar | Author: Varun Pratap Bhardwaj
+License: MIT
+"""
+from __future__ import annotations
+
+import json
+import logging
+import re
+from dataclasses import dataclass
+from typing import Any, Protocol
+
+from superlocalmemory.storage.models import AtomicFact
+
+logger = logging.getLogger(__name__)
+
+_MAX_ROUNDS = 2
+_SUFFICIENCY_SCORE_THRESHOLD = 0.6
+_SKIP_TYPES = frozenset()  # Hotpatch: enable agentic for ALL query types including multi_hop
+
+_SUFFICIENCY_SYSTEM = (
+    "You evaluate whether retrieved context is sufficient to answer a query. "
+    'Respond ONLY with JSON: {"is_sufficient": true/false, "missing_information": "..."}'
+)
+
+_REWRITE_SYSTEM = (
+    "You rewrite queries for a memory retrieval system. "
+    "Respond ONLY with a JSON array of 1-3 rewritten queries: "
+    '["query1", "query2"]'
+)
+
+
+class LLMBackend(Protocol):
+    """Minimal LLM interface."""
+    @property
+    def is_available(self) -> bool: ...
+    def generate(self, prompt: str, system: str = "",
+                 max_tokens: int = 512, temperature: float = 0.0) -> Any: ...
+
+
+class RetrievalEngine(Protocol):
+    """Minimal retrieval engine interface."""
+    def recall_facts(self, query: str, profile_id: str,
+                     top_k: int, skip_agentic: bool = True,
+                     ) -> list[tuple[AtomicFact, float]]: ...
+
+
+class DatabaseProtocol(Protocol):
+    """Minimal DB interface for alias expansion."""
+    def get_entity_by_name(self, name: str, profile_id: str) -> Any: ...
+    def get_aliases_for_entity(self, entity_id: str) -> list[Any]: ...
+
+
+@dataclass
+class RetrievalRound:
+    """Metadata for one retrieval round."""
+    round_num: int
+    query: str
+    result_count: int
+    avg_score: float
+    is_sufficient: bool
+
+
+class AgenticRetriever:
+    """2-round sufficiency verification (EverMemOS pattern).
+
+    Round 1: Retrieve → check sufficiency.
+    Round 2: If insufficient, LLM refines queries → merge → rerank.
+
+    Mode A (no LLM): heuristic alias expansion for round 2.
+    Mode C (LLM): full sufficiency check + query refinement.
+    """
+
+    def __init__(
+        self,
+        confidence_threshold: float = _SUFFICIENCY_SCORE_THRESHOLD,
+        min_results_ratio: float = 0.5,
+        db: DatabaseProtocol | None = None,
+    ) -> None:
+        self._threshold = confidence_threshold
+        self._min_ratio = min_results_ratio
+        self._db = db
+        self.rounds: list[RetrievalRound] = []
+
+    def retrieve(
+        self, query: str, profile_id: str,
+        retrieval_engine: RetrievalEngine,
+        llm: LLMBackend | None = None,
+        top_k: int = 20, query_type: str = "",
+    ) -> list[AtomicFact]:
+        """2-round retrieval with sufficiency check."""
+        self.rounds = []
+
+        # S15: skip agentic for temporal (but NOT multi_hop — bridge handles that)
+        if query_type in _SKIP_TYPES:
+            logger.debug("Skipping agentic for query_type=%s", query_type)
+            return [f for f, _ in retrieval_engine.recall_facts(
+                query, profile_id, top_k=top_k, skip_agentic=True)]
+
+        # Round 1: standard retrieval
+        r1 = retrieval_engine.recall_facts(
+            query, profile_id, top_k=top_k, skip_agentic=True,
+        )
+        r1_avg = _avg(r1)
+        max_score = max((s for _, s in r1), default=0.0)
+
+        # Sufficiency check
+        is_sufficient = self._check_sufficiency(query, r1, llm)
+        self.rounds.append(RetrievalRound(1, query, len(r1), r1_avg, is_sufficient))
+
+        # Return if sufficient OR no way to improve (no LLM and no DB)
+        if is_sufficient:
+            return [f for f, _ in r1[:top_k]]
+
+        # Trigger round 2 only when: low score OR multi_hop
+        needs_round2 = (
+            max_score < self._threshold
+            or query_type == "multi_hop"
+            or len(r1) < 3
+        )
+        if not needs_round2:
+            return [f for f, _ in r1[:top_k]]
+
+        # Round 2: refinement
+        pool: dict[str, tuple[AtomicFact, float]] = {
+            f.fact_id: (f, s) for f, s in r1
+        }
+
+        if llm is not None and getattr(llm, "is_available", False):
+            # Mode C: LLM generates refined queries
+            refined = self._llm_refine(query, r1, llm)
+        else:
+            # Mode A: heuristic alias expansion
+            refined = self._heuristic_expand(query, profile_id)
+
+        for rq in refined:
+            rn = retrieval_engine.recall_facts(
+                rq, profile_id, top_k=top_k, skip_agentic=True,
+            )
+            for fact, score in rn:
+                existing = pool.get(fact.fact_id)
+                if existing is None or score > existing[1]:
+                    pool[fact.fact_id] = (fact, score)
+            self.rounds.append(
+                RetrievalRound(2, rq, len(rn), _avg(rn), True),
+            )
+
+        merged = sorted(pool.values(), key=lambda x: x[1], reverse=True)
+        return [f for f, _ in merged[:top_k]]
+
+    # -- Sufficiency check ---------------------------------------------------
+
+    def _check_sufficiency(
+        self, query: str, results: list[tuple[AtomicFact, float]],
+        llm: LLMBackend | None,
+    ) -> bool:
+        """Three-way sufficiency: SUFFICIENT / INSUFFICIENT / AMBIGUOUS."""
+        if not results:
+            return False
+
+        max_score = max((s for _, s in results), default=0.0)
+
+        # Heuristic fast path: clearly sufficient
+        if max_score >= 0.8 and len(results) >= 5:
+            return True
+
+        # Heuristic fast path: clearly insufficient
+        if max_score < 0.3 or len(results) < 2:
+            return False
+
+        # LLM sufficiency check (Mode C only)
+        if llm is not None and getattr(llm, "is_available", False):
+            try:
+                top5_context = "\n".join(
+                    f"- {f.content}" for f, _ in results[:5]
+                )
+                prompt = (
+                    f"Query: {query}\n\n"
+                    f"Retrieved context:\n{top5_context}\n\n"
+                    "Is this context sufficient to answer the query?"
+                )
+                resp = llm.generate(
+                    prompt=prompt, system=_SUFFICIENCY_SYSTEM,
+                    max_tokens=128, temperature=0.0,
+                )
+                text = getattr(resp, "text", str(resp))
+                parsed = _parse_sufficiency(text)
+                if parsed is not None:
+                    return parsed
+            except Exception as exc:
+                logger.warning("Sufficiency check failed: %s", exc)
+
+        # Default: sufficient if score is above threshold
+        return max_score >= self._threshold
+
+    # -- Query refinement ----------------------------------------------------
+
+    @staticmethod
+    def _llm_refine(
+        query: str,
+        prev: list[tuple[AtomicFact, float]],
+        llm: LLMBackend,
+    ) -> list[str]:
+        """LLM generates 2-3 refined queries from missing information."""
+        ctx = ""
+        if prev:
+            ctx = f"\nCurrent results: {[f.content[:80] for f, _ in prev[:3]]}"
+        try:
+            resp = llm.generate(
+                prompt=(
+                    f"Original query: {query}\n"
+                    f"Insufficient results.{ctx}\n"
+                    "Generate 2-3 refined search queries to find missing information."
+                ),
+                system=_REWRITE_SYSTEM,
+                max_tokens=256,
+                temperature=0.0,
+            )
+            parsed = _parse_json_strings(getattr(resp, "text", str(resp)))
+            if parsed:
+                return parsed[:3]
+        except Exception as exc:
+            logger.warning("LLM refine failed: %s", exc)
+        return []
+
+    def _heuristic_expand(
+        self, query: str, profile_id: str,
+    ) -> list[str]:
+        """Mode A: expand query with entity aliases (no LLM)."""
+        if self._db is None:
+            return []
+
+        expanded_parts: list[str] = []
+        entities = re.findall(r"\b[A-Z][a-z]{2,}\b", query)
+        for name in entities:
+            entity = self._db.get_entity_by_name(name, profile_id)
+            if entity:
+                aliases = self._db.get_aliases_for_entity(entity.entity_id)
+                for a in aliases[:3]:
+                    expanded_parts.append(a.alias)
+
+        if expanded_parts:
+            return [query + " " + " ".join(expanded_parts)]
+        return []
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+def _avg(results: list[tuple[AtomicFact, float]]) -> float:
+    return sum(s for _, s in results) / len(results) if results else 0.0
+
+
+def _parse_json_strings(raw: str) -> list[str]:
+    """Extract JSON string array from LLM output."""
+    try:
+        m = re.search(r"\[.*\]", raw, re.DOTALL)
+        if not m:
+            return []
+        items = json.loads(m.group())
+        if isinstance(items, list):
+            return [str(q).strip() for q in items[:3] if q]
+        return []
+    except (json.JSONDecodeError, ValueError):
+        return []
+
+
+def _parse_sufficiency(raw: str) -> bool | None:
+    """Parse LLM sufficiency response JSON."""
+    try:
+        m = re.search(r"\{.*\}", raw, re.DOTALL)
+        if not m:
+            return None
+        data = json.loads(m.group())
+        if isinstance(data, dict) and "is_sufficient" in data:
+            return bool(data["is_sufficient"])
+        return None
+    except (json.JSONDecodeError, ValueError):
+        return None

package/src/superlocalmemory/retrieval/ann_index.py

@@ -0,0 +1,223 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under the MIT License - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""SuperLocalMemory V3 — Approximate Nearest Neighbor Index.
+
+Numpy-based cosine similarity with thread-safe operations. Supports
+rebuilding from database on cold start and incremental add/remove.
+
+No FAISS dependency — pure numpy brute-force is sufficient for the
+scale we target (up to 100K facts). At that scale, a single cosine
+scan takes ~5ms on CPU which is well within our latency budget.
+
+Part of Qualixar | Author: Varun Pratap Bhardwaj
+License: MIT
+"""
+
+from __future__ import annotations
+
+import logging
+import threading
+
+import numpy as np
+
+logger = logging.getLogger(__name__)
+
+
+class ANNIndex:
+    """Thread-safe approximate nearest neighbor index using numpy.
+
+    Stores (fact_id, embedding) pairs and supports top-k cosine
+    similarity search. Vectors are L2-normalized on insertion for
+    efficient dot-product scoring.
+
+    Args:
+        dimension: Embedding vector dimension (e.g. 768 for nomic-embed).
+    """
+
+    def __init__(self, dimension: int) -> None:
+        self._dim = dimension
+        self._ids: list[str] = []
+        self._id_to_idx: dict[str, int] = {}
+        self._vectors: list[np.ndarray] = []
+        self._lock = threading.Lock()
+
+    # ------------------------------------------------------------------
+    # Properties
+    # ------------------------------------------------------------------
+
+    @property
+    def size(self) -> int:
+        """Number of indexed vectors."""
+        with self._lock:
+            return len(self._ids)
+
+    @property
+    def dimension(self) -> int:
+        """Embedding dimension this index was created for."""
+        return self._dim
+
+    # ------------------------------------------------------------------
+    # Mutation
+    # ------------------------------------------------------------------
+
+    def add(self, fact_id: str, embedding: list[float]) -> None:
+        """Add or update a fact embedding in the index.
+
+        The vector is L2-normalized before storage so that dot-product
+        equals cosine similarity at search time.
+
+        Args:
+            fact_id: Unique fact identifier.
+            embedding: Raw embedding vector (will be normalized).
+        """
+        vec = np.asarray(embedding, dtype=np.float32).ravel()
+        if vec.shape[0] != self._dim:
+            logger.warning(
+                "Dimension mismatch: expected %d, got %d for %s",
+                self._dim, vec.shape[0], fact_id,
+            )
+            return
+
+        norm = np.linalg.norm(vec)
+        if norm > 1e-10:
+            vec = vec / norm
+
+        with self._lock:
+            if fact_id in self._id_to_idx:
+                # Update existing entry
+                idx = self._id_to_idx[fact_id]
+                self._vectors[idx] = vec
+            else:
+                # Append new entry
+                self._id_to_idx[fact_id] = len(self._ids)
+                self._ids.append(fact_id)
+                self._vectors.append(vec)
+
+    def remove(self, fact_id: str) -> None:
+        """Remove a fact from the index.
+
+        Uses swap-and-pop for O(1) removal: the last element fills
+        the gap left by the removed element.
+
+        Args:
+            fact_id: Fact identifier to remove. No-op if not found.
+        """
+        with self._lock:
+            if fact_id not in self._id_to_idx:
+                return
+
+            idx = self._id_to_idx.pop(fact_id)
+            last_idx = len(self._ids) - 1
+
+            if idx != last_idx:
+                # Swap with last element
+                last_id = self._ids[last_idx]
+                self._ids[idx] = last_id
+                self._vectors[idx] = self._vectors[last_idx]
+                self._id_to_idx[last_id] = idx
+
+            self._ids.pop()
+            self._vectors.pop()
+
+    def clear(self) -> None:
+        """Remove all indexed vectors."""
+        with self._lock:
+            self._ids.clear()
+            self._id_to_idx.clear()
+            self._vectors.clear()
+
+    # ------------------------------------------------------------------
+    # Search
+    # ------------------------------------------------------------------
+
+    def search(
+        self,
+        query_embedding: list[float],
+        top_k: int = 30,
+    ) -> list[tuple[str, float]]:
+        """Find top-k most similar facts by cosine similarity.
+
+        Args:
+            query_embedding: Query vector (will be L2-normalized).
+            top_k: Number of results to return.
+
+        Returns:
+            List of (fact_id, score) tuples sorted by score descending.
+            Score is cosine similarity in [-1.0, 1.0].
+        """
+        q = np.asarray(query_embedding, dtype=np.float32).ravel()
+        if q.shape[0] != self._dim:
+            logger.warning(
+                "Query dim mismatch: expected %d, got %d",
+                self._dim, q.shape[0],
+            )
+            return []
+
+        norm = np.linalg.norm(q)
+        if norm < 1e-10:
+            return []
+        q_normed = q / norm
+
+        with self._lock:
+            if not self._vectors:
+                return []
+
+            # Stack into matrix for vectorized dot product
+            mat = np.stack(self._vectors)        # shape: (N, dim)
+            scores = mat @ q_normed              # shape: (N,)
+
+            # Partial sort for top-k (faster than full sort for large N)
+            k = min(top_k, len(scores))
+            if k <= 0:
+                return []
+
+            top_indices = np.argpartition(scores, -k)[-k:]
+            top_indices = top_indices[np.argsort(scores[top_indices])[::-1]]
+
+            return [
+                (self._ids[i], float(scores[i]))
+                for i in top_indices
+            ]
+
+    # ------------------------------------------------------------------
+    # Bulk loading (cold start)
+    # ------------------------------------------------------------------
+
+    def rebuild(
+        self,
+        fact_ids: list[str],
+        embeddings: list[list[float]],
+    ) -> int:
+        """Rebuild the entire index from database contents.
+
+        Replaces all existing entries. Used on cold start to populate
+        the index from persisted embeddings.
+
+        Args:
+            fact_ids: List of fact identifiers.
+            embeddings: Corresponding embedding vectors.
+
+        Returns:
+            Number of vectors successfully indexed.
+        """
+        if len(fact_ids) != len(embeddings):
+            logger.error(
+                "rebuild: mismatched lengths — %d ids vs %d embeddings",
+                len(fact_ids), len(embeddings),
+            )
+            return 0
+
+        with self._lock:
+            self._ids.clear()
+            self._id_to_idx.clear()
+            self._vectors.clear()
+
+        indexed = 0
+        for fid, emb in zip(fact_ids, embeddings):
+            self.add(fid, emb)
+            indexed += 1
+
+        logger.info("ANN index rebuilt with %d vectors (dim=%d)", indexed, self._dim)
+        return indexed

package/src/superlocalmemory/retrieval/bm25_channel.py

@@ -0,0 +1,185 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under the MIT License - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""SuperLocalMemory V3 — BM25 Keyword Search Channel.
+
+Persistent BM25Plus index over fact content. Catches exact name/date
+matches that embedding similarity misses.
+
+V1 bug fix: V1 kept BM25 tokens in-memory only — a restart lost
+the entire index. This version persists tokens to the DB via
+store_bm25_tokens / get_all_bm25_tokens and cold-loads on init.
+
+Part of Qualixar | Author: Varun Pratap Bhardwaj
+License: MIT
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+from typing import TYPE_CHECKING
+
+from rank_bm25 import BM25Plus
+
+if TYPE_CHECKING:
+    from superlocalmemory.storage.database import DatabaseManager
+
+logger = logging.getLogger(__name__)
+
+# Minimal stopwords — small set to avoid stripping important terms
+_STOPWORDS: frozenset[str] = frozenset({
+    "a", "an", "the", "is", "are", "was", "were", "be", "been", "being",
+    "have", "has", "had", "do", "does", "did", "will", "would", "shall",
+    "should", "may", "might", "must", "can", "could", "of", "in", "to",
+    "for", "with", "on", "at", "from", "by", "as", "into", "through",
+    "and", "but", "or", "nor", "not", "so", "yet", "if", "then", "than",
+    "that", "this", "it", "its", "i", "me", "my", "we", "our", "you",
+    "your", "he", "him", "his", "she", "her", "they", "them", "their",
+})
+
+# Token pattern: words with letters/digits, keeps hyphens and apostrophes
+_TOKEN_RE = re.compile(r"[a-zA-Z0-9][\w'-]*[a-zA-Z0-9]|[a-zA-Z0-9]")
+
+
+def tokenize(text: str) -> list[str]:
+    """Tokenize text: lowercase, split, remove stopwords.
+
+    Exported so encoding pipeline can persist tokens at ingest time.
+    """
+    tokens = _TOKEN_RE.findall(text.lower())
+    return [t for t in tokens if t not in _STOPWORDS]
+
+
+class BM25Channel:
+    """Persistent BM25Plus index for keyword retrieval.
+
+    On cold start, loads all tokens from the DB. After that, new facts
+    are added incrementally. The BM25Plus model is rebuilt lazily
+    before each search when the corpus has changed.
+
+    Attributes:
+        document_count: Number of indexed documents.
+    """
+
+    def __init__(self, db: DatabaseManager) -> None:
+        self._db = db
+        self._corpus: list[list[str]] = []
+        self._fact_ids: list[str] = []
+        self._fact_id_set: set[str] = set()
+        self._bm25: BM25Plus | None = None
+        self._dirty: bool = False
+        self._loaded_profiles: set[str] = set()
+
+    @property
+    def document_count(self) -> int:
+        return len(self._corpus)
+
+    def ensure_loaded(self, profile_id: str) -> None:
+        """Cold-load BM25 tokens from DB for a profile (once).
+
+        Idempotent: subsequent calls for the same profile are no-ops.
+        """
+        if profile_id in self._loaded_profiles:
+            return
+
+        token_map = self._db.get_all_bm25_tokens(profile_id)
+        if not token_map:
+            # Fallback: tokenize facts directly if no pre-stored tokens
+            facts = self._db.get_all_facts(profile_id)
+            for fact in facts:
+                if fact.fact_id in self._fact_id_set:
+                    continue
+                tokens = tokenize(fact.content)
+                if tokens:
+                    self._corpus.append(tokens)
+                    self._fact_ids.append(fact.fact_id)
+                    self._fact_id_set.add(fact.fact_id)
+                    # Persist for next cold start
+                    self._db.store_bm25_tokens(fact.fact_id, profile_id, tokens)
+        else:
+            for fid, tokens in token_map.items():
+                if fid in self._fact_id_set:
+                    continue
+                self._corpus.append(tokens)
+                self._fact_ids.append(fid)
+                self._fact_id_set.add(fid)
+
+        self._dirty = True
+        self._loaded_profiles.add(profile_id)
+        logger.debug(
+            "BM25 cold-loaded %d documents for profile=%s",
+            len(token_map) if token_map else 0, profile_id,
+        )
+
+    def add(self, fact_id: str, content: str, profile_id: str) -> None:
+        """Add a single fact to the index and persist tokens.
+
+        Args:
+            fact_id: Unique fact identifier.
+            content: Raw text content to index.
+            profile_id: Owner profile.
+        """
+        tokens = tokenize(content)
+        if not tokens:
+            return
+
+        self._corpus.append(tokens)
+        self._fact_ids.append(fact_id)
+        self._fact_id_set.add(fact_id)
+        self._dirty = True
+
+        # Persist for cold start
+        self._db.store_bm25_tokens(fact_id, profile_id, tokens)
+
+    def search(
+        self,
+        query: str,
+        profile_id: str,
+        top_k: int = 30,
+    ) -> list[tuple[str, float]]:
+        """Search BM25 index for matching facts.
+
+        Auto-loads from DB on first call for this profile.
+
+        Args:
+            query: Search query text.
+            profile_id: Scope to this profile.
+            top_k: Maximum results.
+
+        Returns:
+            List of (fact_id, bm25_score) sorted by score descending.
+        """
+        self.ensure_loaded(profile_id)
+
+        if not self._corpus:
+            return []
+
+        query_tokens = tokenize(query)
+        if not query_tokens:
+            return []
+
+        # Rebuild BM25 model if corpus changed
+        if self._dirty or self._bm25 is None:
+            self._bm25 = BM25Plus(self._corpus, k1=1.2, b=0.75)
+            self._dirty = False
+
+        scores = self._bm25.get_scores(query_tokens)
+
+        scored: list[tuple[str, float]] = []
+        for i, score in enumerate(scores):
+            if score > 0.0:
+                scored.append((self._fact_ids[i], float(score)))
+
+        scored.sort(key=lambda x: x[1], reverse=True)
+        return scored[:top_k]
+
+    def clear(self) -> None:
+        """Clear the in-memory index (does NOT delete DB tokens)."""
+        self._corpus = []
+        self._fact_ids = []
+        self._fact_id_set = set()
+        self._bm25 = None
+        self._dirty = False
+        self._loaded_profiles = set()