superlocalmemory 2.8.5 → 3.0.0

package/src/superlocalmemory/retrieval/entity_channel.py

@@ -0,0 +1,179 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under the MIT License - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""SuperLocalMemory V3 — Entity Graph Channel with Spreading Activation.
+
+SA-RAG pattern: entities from query -> canonical lookup -> graph traversal
+with decay. Handles BOTH uppercase and lowercase entity mentions.
+
+Part of Qualixar | Author: Varun Pratap Bhardwaj
+License: MIT
+"""
+from __future__ import annotations
+
+import json
+import logging
+import re
+from collections import defaultdict
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from superlocalmemory.encoding.entity_resolver import EntityResolver
+    from superlocalmemory.storage.database import DatabaseManager
+
+logger = logging.getLogger(__name__)
+
+_PROPER_NOUN_RE = re.compile(r"\b[A-Z][a-z]{1,}\b")
+
+_ENTITY_STOP: frozenset[str] = frozenset({
+    # Expanded stop list for query entity extraction
+    "what", "when", "where", "who", "which", "how", "does", "did",
+    "the", "that", "this", "there", "then", "than", "they", "them",
+    "have", "has", "had", "been", "being", "about", "after", "before",
+    "from", "into", "with", "some", "other", "would", "could", "should",
+    "will", "because", "also", "just", "like", "know", "think",
+    "feel", "want", "need", "make", "take", "give", "tell", "said",
+    "wow", "gonna", "got", "by", "thanks", "thank", "hey", "hi",
+    "hello", "bye", "good", "great", "nice", "cool", "right",
+    "let", "can", "might", "much", "many", "more", "most",
+    "something", "anything", "everything", "nothing", "someone",
+    "it", "my", "your", "our", "their", "me", "you", "we", "us",
+    "do", "if", "or", "no", "to", "at", "on", "in", "so",
+    "go", "come", "see", "look", "say", "ask", "try", "keep",
+    "yes", "yeah", "sure", "okay", "ok", "really", "actually",
+    "maybe", "well", "still", "even", "very",
+})
+
+
+def extract_query_entities(query: str) -> list[str]:
+    """Extract entity candidates from query (handles both cases).
+
+    Strategy: find proper nouns in original + title-cased text,
+    plus quoted phrases. Deduplicates case-insensitively.
+    """
+    candidates: list[str] = []
+    seen: set[str] = set()
+
+    def _add(name: str) -> None:
+        lo = name.lower()
+        if lo not in seen and lo not in _ENTITY_STOP and len(name) >= 2:
+            seen.add(lo)
+            candidates.append(name)
+
+    for m in _PROPER_NOUN_RE.finditer(query):
+        _add(m.group(0))
+    for m in _PROPER_NOUN_RE.finditer(query.title()):
+        _add(m.group(0))
+    for m in re.finditer(r'"([^"]+)"', query):
+        _add(m.group(1).strip())
+
+    return candidates
+
+
+class EntityGraphChannel:
+    """Entity-based retrieval with spreading activation (SA-RAG)."""
+
+    def __init__(
+        self, db: DatabaseManager,
+        entity_resolver: EntityResolver | None = None,
+        decay: float = 0.7, activation_threshold: float = 0.1,
+        max_hops: int = 3,
+    ) -> None:
+        self._db = db
+        self._resolver = entity_resolver
+        self._decay = decay
+        self._threshold = activation_threshold
+        self._max_hops = max_hops
+
+    def search(self, query: str, profile_id: str, top_k: int = 50) -> list[tuple[str, float]]:
+        """Search via entity graph with spreading activation."""
+        raw_entities = extract_query_entities(query)
+        if not raw_entities:
+            return []
+
+        canonical_ids = self._resolve_entities(raw_entities, profile_id)
+        if not canonical_ids:
+            return []
+
+        # Seed activation from direct entity-linked facts
+        activation: dict[str, float] = defaultdict(float)
+        visited_entities: set[str] = set(canonical_ids)
+
+        for eid in canonical_ids:
+            for fact in self._db.get_facts_by_entity(eid, profile_id):
+                activation[fact.fact_id] = max(activation[fact.fact_id], 1.0)
+
+        # Spreading activation through graph edges
+        frontier = set(activation.keys())
+        for hop in range(1, self._max_hops):
+            hop_decay = self._decay ** hop
+            if hop_decay < self._threshold:
+                break
+            next_frontier: set[str] = set()
+
+            for fid in frontier:
+                for edge in self._db.get_edges_for_node(fid, profile_id):
+                    neighbor = edge.target_id if edge.source_id == fid else edge.source_id
+                    propagated = activation[fid] * self._decay
+                    if propagated >= self._threshold and propagated > activation.get(neighbor, 0.0):
+                        activation[neighbor] = propagated
+                        next_frontier.add(neighbor)
+
+            # Discover new entities from activated facts -> get their facts
+            new_eids = self._discover_entities(frontier, profile_id, visited_entities)
+            for eid in new_eids:
+                visited_entities.add(eid)
+                for fact in self._db.get_facts_by_entity(eid, profile_id):
+                    if hop_decay > activation.get(fact.fact_id, 0.0):
+                        activation[fact.fact_id] = hop_decay
+                        next_frontier.add(fact.fact_id)
+
+            frontier = next_frontier
+            if not frontier:
+                break
+
+        results = [(fid, sc) for fid, sc in activation.items() if sc >= self._threshold]
+        results.sort(key=lambda x: x[1], reverse=True)
+        return results[:top_k]
+
+    def _resolve_entities(self, raw: list[str], profile_id: str) -> list[str]:
+        """Resolve raw names to canonical entity IDs."""
+        ids: list[str] = []
+        seen: set[str] = set()
+        if self._resolver is not None:
+            for eid in self._resolver.resolve(raw, profile_id).values():
+                if eid not in seen:
+                    seen.add(eid)
+                    ids.append(eid)
+        else:
+            for name in raw:
+                ent = self._db.get_entity_by_name(name, profile_id)
+                if ent and ent.entity_id not in seen:
+                    seen.add(ent.entity_id)
+                    ids.append(ent.entity_id)
+        return ids
+
+    def _discover_entities(
+        self, fact_ids: set[str], profile_id: str, visited: set[str],
+    ) -> list[str]:
+        """Find new canonical entity IDs referenced by a set of facts."""
+        new: list[str] = []
+        seen = set(visited)
+        for fid in fact_ids:
+            rows = self._db.execute(
+                "SELECT canonical_entities_json FROM atomic_facts WHERE fact_id = ?", (fid,),
+            )
+            if not rows:
+                continue
+            raw = dict(rows[0]).get("canonical_entities_json")
+            if not raw:
+                continue
+            try:
+                for eid in json.loads(raw):
+                    if eid not in seen:
+                        seen.add(eid)
+                        new.append(eid)
+            except (ValueError, TypeError):
+                continue
+        return new

package/src/superlocalmemory/retrieval/fusion.py

@@ -0,0 +1,78 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under the MIT License - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""SuperLocalMemory V3 — Weighted Reciprocal Rank Fusion.
+
+Single-pass RRF with k=60 for diverse retrieval (D116).
+V1 had triple re-fusion which destroyed rankings — fixed in V2.
+
+Part of Qualixar | Author: Varun Pratap Bhardwaj
+License: MIT
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+
+@dataclass(frozen=True)
+class FusionResult:
+    """Single fused result with per-channel provenance."""
+    fact_id: str
+    fused_score: float
+    channel_ranks: dict[str, int] = field(default_factory=dict)
+    channel_scores: dict[str, float] = field(default_factory=dict)
+
+
+def weighted_rrf(
+    channels: dict[str, list[tuple[str, float]]],
+    weights: dict[str, float],
+    k: int = 60,
+    max_rank_penalty: int = 1000,
+) -> list[FusionResult]:
+    """Fuse ranked lists via Weighted Reciprocal Rank Fusion.
+
+    Args:
+        channels: channel_name -> [(fact_id, score)] sorted desc.
+        weights: channel_name -> weight multiplier.
+        k: RRF smoothing constant (60 for diverse retrieval, D116).
+        max_rank_penalty: Rank assigned to absent documents.
+
+    Returns:
+        FusionResult list sorted by fused_score descending.
+    """
+    if k <= 0:
+        raise ValueError(f"k must be positive, got {k}")
+
+    rank_maps: dict[str, dict[str, int]] = {}
+    score_maps: dict[str, dict[str, float]] = {}
+
+    for ch, ranked in channels.items():
+        ranks: dict[str, int] = {}
+        scores: dict[str, float] = {}
+        for i, (fid, sc) in enumerate(ranked):
+            ranks[fid] = i + 1
+            scores[fid] = sc
+        rank_maps[ch] = ranks
+        score_maps[ch] = scores
+
+    all_ids: set[str] = set()
+    for ranked in channels.values():
+        for fid, _ in ranked:
+            all_ids.add(fid)
+
+    results: list[FusionResult] = []
+    for fid in all_ids:
+        fused = 0.0
+        ch_ranks: dict[str, int] = {}
+        ch_scores: dict[str, float] = {}
+        for ch in channels:
+            w = weights.get(ch, 1.0)
+            rank = rank_maps[ch].get(fid, max_rank_penalty)
+            ch_ranks[ch] = rank
+            ch_scores[ch] = score_maps[ch].get(fid, 0.0)
+            fused += w / (k + rank)
+        results.append(FusionResult(fid, fused, ch_ranks, ch_scores))
+
+    results.sort(key=lambda r: r.fused_score, reverse=True)
+    return results

package/src/superlocalmemory/retrieval/profile_channel.py

@@ -0,0 +1,105 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under the MIT License - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""SuperLocalMemory V3 — Profile Channel (Entity-Profile Retrieval).
+
+Returns fact IDs from entity profiles — enables direct answers for
+"What does Alice do?" style queries without full embedding search.
+
+This is a SHORTCUT channel: if the query mentions a known entity,
+the profile's accumulated fact IDs are injected directly into the
+retrieval pool with high scores.
+
+Competitor reference: EverMemOS profile synthesis (~+15-20% SH).
+
+Part of Qualixar | Author: Varun Pratap Bhardwaj
+License: MIT
+"""
+from __future__ import annotations
+
+import logging
+import re
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from superlocalmemory.storage.database import DatabaseManager
+
+logger = logging.getLogger(__name__)
+
+# Pattern for extracting potential entity names (capitalized multi-word)
+_ENTITY_PATTERN = re.compile(r"\b[A-Z][a-z]+(?:\s+[A-Z][a-z]+)*\b")
+
+# Common sentence starters to exclude from entity extraction
+_SENTENCE_STARTERS = frozenset({
+    "What", "Where", "Who", "Which", "How", "When", "Does", "Did",
+    "Can", "Could", "Would", "Should", "Are", "Is", "Was", "Were",
+    "Has", "Have", "The", "Tell", "Please", "Do", "Why",
+})
+
+
+class ProfileChannel:
+    """Entity-profile-based retrieval for direct entity queries.
+
+    If the query mentions a known entity (by canonical name or alias),
+    the entity's profile fact IDs are returned with high base score.
+
+    Usage::
+
+        channel = ProfileChannel(db)
+        results = channel.search("What is Alice's job?", "default")
+        # returns [(fact_id_1, 0.95), (fact_id_2, 0.95), ...]
+    """
+
+    def __init__(self, db: DatabaseManager) -> None:
+        self._db = db
+
+    def search(
+        self,
+        query: str,
+        profile_id: str,
+        top_k: int = 10,
+    ) -> list[tuple[str, float]]:
+        """Search entity profiles for matching facts.
+
+        Args:
+            query: User query text.
+            profile_id: Scope to this profile.
+            top_k: Maximum results to return.
+
+        Returns:
+            List of (fact_id, score) sorted by score descending.
+        """
+        entities = self._extract_entity_names(query)
+        if not entities:
+            return []
+
+        results: list[tuple[str, float]] = []
+        seen: set[str] = set()
+
+        for name in entities:
+            entity = self._db.get_entity_by_name(name, profile_id)
+            if not entity:
+                continue
+
+            profiles = self._db.get_entity_profiles_by_entity(
+                entity.entity_id, profile_id,
+            )
+            for p in profiles:
+                for fid in p.fact_ids:
+                    if fid not in seen:
+                        seen.add(fid)
+                        results.append((fid, 0.95))
+
+        results.sort(key=lambda x: x[1], reverse=True)
+        return results[:top_k]
+
+    @staticmethod
+    def _extract_entity_names(query: str) -> list[str]:
+        """Extract potential entity names from query text.
+
+        Returns capitalized words/phrases that aren't common
+        sentence starters.
+        """
+        matches = _ENTITY_PATTERN.findall(query)
+        return [m for m in matches if m not in _SENTENCE_STARTERS]

package/src/superlocalmemory/retrieval/reranker.py

@@ -0,0 +1,154 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under the MIT License - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""SuperLocalMemory V3 — Cross-Encoder Reranker.
+
+Scores (query, fact) pairs through a cross-encoder in a single forward
+pass. Lazy model loading, thread-safe via lock.
+
+Part of Qualixar | Author: Varun Pratap Bhardwaj
+License: MIT
+"""
+
+from __future__ import annotations
+
+import logging
+import threading
+from typing import Any
+
+from superlocalmemory.storage.models import AtomicFact
+
+logger = logging.getLogger(__name__)
+
+
+class CrossEncoderReranker:
+    """Rerank candidate facts using a local cross-encoder model.
+
+    When the model is unavailable (missing package, download failure,
+    offline environment), falls back to returning candidates in their
+    original score order — never crashes.
+
+    Args:
+        model_name: HuggingFace cross-encoder model identifier.
+    """
+
+    def __init__(
+        self,
+        model_name: str = "BAAI/bge-reranker-v2-m3",
+    ) -> None:
+        self._model_name = model_name
+        self._model: Any = None
+        self._loaded = False
+        self._lock = threading.Lock()
+
+    # ------------------------------------------------------------------
+    # Lazy loading
+    # ------------------------------------------------------------------
+
+    def _ensure_model(self) -> None:
+        """Load cross-encoder on first use (thread-safe)."""
+        if self._loaded:
+            return
+
+        with self._lock:
+            if self._loaded:
+                return  # Double-check after acquiring lock
+            try:
+                from sentence_transformers import CrossEncoder
+
+                self._model = CrossEncoder(self._model_name)
+                logger.info("Cross-encoder loaded: %s", self._model_name)
+            except ImportError:
+                logger.warning(
+                    "sentence-transformers not installed; "
+                    "cross-encoder reranking disabled"
+                )
+            except OSError as exc:
+                logger.warning(
+                    "Failed to load cross-encoder %s: %s",
+                    self._model_name,
+                    exc,
+                )
+            finally:
+                self._loaded = True
+
+    # ------------------------------------------------------------------
+    # Public API
+    # ------------------------------------------------------------------
+
+    def rerank(
+        self,
+        query: str,
+        candidates: list[tuple[AtomicFact, float]],
+        top_k: int = 10,
+    ) -> list[tuple[AtomicFact, float]]:
+        """Rerank candidates by cross-encoder relevance.
+
+        Each (query, fact.content) pair is scored in a single forward
+        pass. Results are returned sorted by cross-encoder score.
+
+        When the model is unavailable, returns candidates sorted by
+        their existing score (graceful fallback).
+
+        Args:
+            query: User query text.
+            candidates: List of (AtomicFact, score) tuples from the
+                fusion stage.
+            top_k: Maximum results to return.
+
+        Returns:
+            Top-k (AtomicFact, cross_encoder_score) tuples, sorted
+            descending by cross-encoder score.
+        """
+        if not candidates:
+            return []
+
+        self._ensure_model()
+
+        if self._model is None:
+            # Fallback: keep existing score order
+            sorted_cands = sorted(
+                candidates, key=lambda x: x[1], reverse=True
+            )
+            return sorted_cands[:top_k]
+
+        # Build (query, document) pairs for batch scoring
+        pairs: list[tuple[str, str]] = [
+            (query, fact.content) for fact, _ in candidates
+        ]
+
+        scores = self._model.predict(pairs)
+
+        scored: list[tuple[AtomicFact, float]] = [
+            (fact, float(score))
+            for (fact, _), score in zip(candidates, scores)
+        ]
+
+        scored.sort(key=lambda x: x[1], reverse=True)
+        return scored[:top_k]
+
+    def score_pair(self, query: str, document: str) -> float:
+        """Score a single (query, document) pair.
+
+        Args:
+            query: Query text.
+            document: Document text.
+
+        Returns:
+            Relevance score (higher = more relevant). 0.0 if model
+            is unavailable.
+        """
+        self._ensure_model()
+
+        if self._model is None:
+            return 0.0
+
+        scores = self._model.predict([(query, document)])
+        return float(scores[0])
+
+    @property
+    def is_available(self) -> bool:
+        """Whether the cross-encoder model is loaded and ready."""
+        self._ensure_model()
+        return self._model is not None