superlocalmemory 2.8.5 → 3.0.0

package/src/superlocalmemory/retrieval/semantic_channel.py

@@ -0,0 +1,232 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under the MIT License - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""SuperLocalMemory V3 — Semantic Retrieval Channel.
+
+Fisher-aware semantic search: uses Fisher-Rao geodesic distance when
+variance data is available, falls back to cosine similarity otherwise.
+
+The Fisher distance is meaningful when memories accumulate evidence
+(repeated confirmation narrows variance). For fresh benchmark data
+where all variances are identical, Fisher distance degenerates to a
+monotonic transform of Euclidean distance — same ranking as cosine.
+
+Part of Qualixar | Author: Varun Pratap Bhardwaj
+License: MIT
+"""
+
+from __future__ import annotations
+
+import logging
+import math
+from typing import TYPE_CHECKING
+
+import numpy as np
+
+if TYPE_CHECKING:
+    from superlocalmemory.storage.database import DatabaseManager
+    from superlocalmemory.storage.models import AtomicFact
+
+logger = logging.getLogger(__name__)
+
+# Minimum variance floor to prevent division-by-zero in Fisher distance
+_VARIANCE_FLOOR: float = 1e-6
+
+
+def _cosine_similarity(a: np.ndarray, b: np.ndarray) -> float:
+    """Cosine similarity in [-1, 1]. Returns 0.0 on zero vectors."""
+    norm_a = np.linalg.norm(a)
+    norm_b = np.linalg.norm(b)
+    if norm_a < _VARIANCE_FLOOR or norm_b < _VARIANCE_FLOOR:
+        return 0.0
+    return float(np.dot(a, b) / (norm_a * norm_b))
+
+
+def _fisher_rao_similarity(
+    mu_q: np.ndarray,
+    mu_f: np.ndarray,
+    var_f: np.ndarray,
+    temperature: float = 15.0,
+) -> float:
+    """Fisher-Rao geodesic similarity on diagonal Gaussian manifold.
+
+    d_FR^2 = sum_i [ (mu_q_i - mu_f_i)^2 / max(var_f_i, eps) ]
+    similarity = exp(-d_FR^2 / temperature)
+
+    When all variances are equal, this is equivalent to a scaled
+    Euclidean distance — same ranking as cosine on normalized vectors.
+    The advantage appears when variances differ across memories.
+    """
+    diff = mu_q - mu_f
+    var_safe = np.maximum(var_f, _VARIANCE_FLOOR)
+    d_sq = float(np.sum(diff * diff / var_safe))
+    return math.exp(-d_sq / temperature)
+
+
+class SemanticChannel:
+    """Dense semantic retrieval via embedding similarity.
+
+    Scans all facts for a profile. Uses a GRADUATED Fisher-Rao ramp:
+    fresh facts (low access_count) use cosine, frequently-accessed facts
+    transition to Fisher-Rao distance for uncertainty-aware similarity.
+
+    Graduated ramp: weight = min(1.2, access_count / 10 * 1.2)
+    Final sim = fisher_weight * fisher_sim + (1 - fisher_weight) * cosine_sim
+    """
+
+    def __init__(
+        self,
+        db: DatabaseManager,
+        fisher_temperature: float = 15.0,
+        embedder: object | None = None,
+        fisher_mode: str = "simplified",
+    ) -> None:
+        self._db = db
+        self._temperature = fisher_temperature
+        self._embedder = embedder
+        self._fisher_mode = fisher_mode if fisher_mode in ("simplified", "full") else "simplified"
+        # Lazily instantiated full metric (avoids import cost when not needed)
+        self._full_metric: object | None = None
+
+    def search(
+        self,
+        query_embedding: list[float],
+        profile_id: str,
+        top_k: int = 50,
+    ) -> list[tuple[str, float]]:
+        """Search for semantically similar facts.
+
+        Uses graduated Fisher-Rao ramp: access_count < 1 = pure cosine,
+        access_count >= 10 = full Fisher-Rao (1.2x weight).
+
+        Args:
+            query_embedding: Dense vector for the query.
+            profile_id: Scope to this profile.
+            top_k: Maximum results to return.
+
+        Returns:
+            List of (fact_id, score) sorted by score descending.
+            Score is in [0, 1] range.
+        """
+        if not query_embedding:
+            return []
+
+        q_vec = np.array(query_embedding, dtype=np.float32)
+
+        # Compute query Fisher params for Bayesian comparison (F45 fix)
+        q_mean: np.ndarray | None = None
+        q_var: np.ndarray | None = None
+        if self._embedder and hasattr(self._embedder, 'compute_fisher_params'):
+            qm, qv = self._embedder.compute_fisher_params(query_embedding)
+            q_mean = np.array(qm, dtype=np.float32)
+            q_var = np.array(qv, dtype=np.float32)
+
+        facts = self._db.get_all_facts(profile_id)
+
+        scored: list[tuple[str, float]] = []
+        for fact in facts:
+            if fact.embedding is None:
+                continue
+
+            f_vec = np.array(fact.embedding, dtype=np.float32)
+            if f_vec.shape != q_vec.shape:
+                continue
+
+            # Cosine baseline (always computed)
+            cos_sim = (_cosine_similarity(q_vec, f_vec) + 1.0) / 2.0
+
+            # Graduated Fisher-Rao ramp (F37, F108)
+            fisher_weight = min(1.2, (fact.access_count or 0) / 10.0 * 1.2)
+
+            if (fisher_weight > 0.01
+                    and fact.fisher_variance is not None
+                    and len(fact.fisher_variance) == len(q_vec)):
+                var_vec = np.array(fact.fisher_variance, dtype=np.float32)
+                f_sim = self._compute_fisher_sim(
+                    q_vec, f_vec, var_vec, fact, q_mean, q_var,
+                )
+                capped_w = min(1.0, fisher_weight)
+                sim = capped_w * f_sim + (1.0 - capped_w) * cos_sim
+            else:
+                sim = cos_sim
+
+            if sim > 0.3:
+                scored.append((fact.fact_id, sim))
+
+        scored.sort(key=lambda x: x[1], reverse=True)
+        return scored[:top_k]
+
+    # ------------------------------------------------------------------
+    # Fisher similarity dispatch
+    # ------------------------------------------------------------------
+
+    def _compute_fisher_sim(
+        self,
+        q_vec: np.ndarray,
+        f_vec: np.ndarray,
+        var_vec: np.ndarray,
+        fact: AtomicFact,
+        q_mean: np.ndarray | None,
+        q_var: np.ndarray | None,
+    ) -> float:
+        """Compute Fisher-Rao similarity using simplified or full metric.
+
+        Simplified (default): Mahalanobis-like distance using only fact variance.
+        Full: Atkinson-Mitchell geodesic via FisherRaoMetric.similarity(),
+              requires both query and fact (mean, variance) pairs.
+
+        Falls back to simplified if full metric cannot be applied (e.g.
+        missing fisher_mean on the fact, or missing query variance).
+        """
+        if self._fisher_mode == "full":
+            return self._compute_full_fisher_sim(
+                q_vec, f_vec, var_vec, fact, q_mean, q_var,
+            )
+        return _fisher_rao_similarity(q_vec, f_vec, var_vec, self._temperature)
+
+    def _compute_full_fisher_sim(
+        self,
+        q_vec: np.ndarray,
+        f_vec: np.ndarray,
+        var_vec: np.ndarray,
+        fact: AtomicFact,
+        q_mean: np.ndarray | None,
+        q_var: np.ndarray | None,
+    ) -> float:
+        """Full Atkinson-Mitchell geodesic via FisherRaoMetric.
+
+        Requires fisher_mean on the fact AND query variance. If either is
+        missing, falls back to the simplified local computation so that
+        the graduated ramp still produces a score.
+        """
+        # Need fact fisher_mean for the full metric
+        fact_mean = fact.fisher_mean
+        if fact_mean is None or len(fact_mean) != len(f_vec):
+            # No stored mean — fall back to simplified
+            return _fisher_rao_similarity(q_vec, f_vec, var_vec, self._temperature)
+
+        # Need query variance for the full metric
+        if q_mean is None or q_var is None:
+            return _fisher_rao_similarity(q_vec, f_vec, var_vec, self._temperature)
+
+        if len(q_mean) != len(fact_mean) or len(q_var) != len(var_vec):
+            return _fisher_rao_similarity(q_vec, f_vec, var_vec, self._temperature)
+
+        metric = self._get_full_metric()
+        try:
+            return metric.similarity(
+                q_mean.tolist(), q_var.tolist(),
+                list(fact_mean), list(var_vec),
+            )
+        except (ValueError, FloatingPointError):
+            # Numerical issue — fall back gracefully
+            logger.debug("Full Fisher metric raised; falling back to simplified")
+            return _fisher_rao_similarity(q_vec, f_vec, var_vec, self._temperature)
+
+    def _get_full_metric(self) -> "FisherRaoMetric":  # noqa: F821
+        """Lazy-load FisherRaoMetric to avoid import-time cost."""
+        if self._full_metric is None:
+            from superlocalmemory.math.fisher import FisherRaoMetric
+            self._full_metric = FisherRaoMetric(temperature=self._temperature)
+        return self._full_metric  # type: ignore[return-value]

package/src/superlocalmemory/retrieval/strategy.py

@@ -0,0 +1,96 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under the MIT License - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""SuperLocalMemory V3 — Query-Adaptive Strategy.
+
+Classifies query type and returns per-type channel weights.
+V1 had this code (strategy_learner.py) but never wired it in.
+
+Part of Qualixar | Author: Varun Pratap Bhardwaj
+License: MIT
+"""
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+
+STRATEGY_PRESETS: dict[str, dict[str, float]] = {
+    "temporal": {"semantic": 0.8, "bm25": 1.5, "entity_graph": 0.8, "temporal": 2.0},
+    "multi_hop": {"semantic": 1.0, "bm25": 0.8, "entity_graph": 2.0, "temporal": 0.5},
+    "aggregation": {"semantic": 1.2, "bm25": 1.5, "entity_graph": 1.0, "temporal": 0.5},
+    "opinion": {"semantic": 1.8, "bm25": 0.6, "entity_graph": 0.8, "temporal": 0.3},
+    "factual": {"semantic": 1.2, "bm25": 1.4, "entity_graph": 1.0, "temporal": 0.6},
+    "entity": {"semantic": 1.0, "bm25": 1.5, "entity_graph": 1.2, "temporal": 0.5},
+    "general": {},
+}
+
+_TEMPORAL_WORDS: frozenset[str] = frozenset({
+    "when", "date", "time", "year", "month", "ago", "before", "after",
+    "during", "last", "next", "recently", "earlier", "later", "since",
+    "until", "while", "between", "january", "february", "march",
+    "april", "may", "june", "july", "august", "september", "october",
+    "november", "december",
+})
+
+_MULTI_HOP_PHRASES: tuple[str, ...] = (
+    "and then", "after that", "because", "how did",
+    "as a result", "led to", "connection between", "relationship between",
+)
+
+_AGGREGATION_WORDS: frozenset[str] = frozenset({
+    "all", "list", "every", "everything", "various", "different",
+    "many", "several", "multiple", "summarize", "overview",
+})
+
+_OPINION_WORDS: tuple[str, ...] = (
+    "think", "feel", "opinion", "prefer", "favorite", "best", "worst",
+    "believe", "like about", "dislike", "enjoy", "hate", "love",
+)
+
+
+@dataclass
+class QueryStrategy:
+    """Classified query type + adapted weights."""
+    query_type: str = "general"
+    weights: dict[str, float] = field(default_factory=dict)
+    confidence: float = 0.5
+
+
+class QueryStrategyClassifier:
+    """Classifies queries and produces adaptive channel weights."""
+
+    def classify(self, query: str, base_weights: dict[str, float]) -> QueryStrategy:
+        """Classify query and return adapted weights."""
+        qtype = self._detect_type(query)
+        adapted = dict(base_weights)
+        for ch, w in STRATEGY_PRESETS.get(qtype, {}).items():
+            adapted[ch] = base_weights.get(ch, 1.0) * w
+        return QueryStrategy(qtype, adapted, 0.7 if qtype != "general" else 0.5)
+
+    def _detect_type(self, query: str) -> str:
+        q = query.lower()
+        # Strip punctuation from words so "january?" matches "january"
+        words = set(re.sub(r"[^\w\s'-]", "", q).split())
+
+        # Check multi_hop BEFORE temporal — phrases like "connection between"
+        # must not be short-circuited by the word "between" in _TEMPORAL_WORDS.
+        if any(p in q for p in _MULTI_HOP_PHRASES):
+            return "multi_hop"
+        if words & _TEMPORAL_WORDS:
+            return "temporal"
+        if words & _AGGREGATION_WORDS:
+            return "aggregation"
+        if any(w in q for w in _OPINION_WORDS):
+            return "opinion"
+        # Proper nouns — exclude common sentence-initial words
+        _SENTENCE_STARTERS = {"What", "Where", "Who", "Which", "How", "When",
+                              "Does", "Did", "Can", "Could", "Would", "Should",
+                              "Are", "Is", "Was", "Were", "Has", "Have", "The", "Tell"}
+        proper_nouns = [m for m in re.findall(r"\b[A-Z][a-z]{1,}\b", query)
+                        if m not in _SENTENCE_STARTERS]
+        if len(proper_nouns) >= 2:
+            return "entity"
+        if q.startswith(("what ", "where ", "who ", "which ", "how ")):
+            return "factual"
+        return "general"

package/src/superlocalmemory/retrieval/temporal_channel.py

@@ -0,0 +1,175 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under the MIT License - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""SuperLocalMemory V3 — Temporal Retrieval Channel (3-Date Model).
+
+Searches by referenced_date (NOT just created_at like V1).
+Returns empty when query has no temporal signal (no recency noise).
+
+Part of Qualixar | Author: Varun Pratap Bhardwaj
+License: MIT
+"""
+from __future__ import annotations
+
+import logging
+import math
+from datetime import datetime
+from typing import TYPE_CHECKING
+
+from dateutil.parser import parse as dateutil_parse, ParserError
+
+from superlocalmemory.encoding.temporal_parser import TemporalParser
+
+if TYPE_CHECKING:
+    from superlocalmemory.storage.database import DatabaseManager
+
+logger = logging.getLogger(__name__)
+
+_MAX_PROXIMITY_DAYS: float = 365.0
+
+
+def _parse_iso(s: str | None) -> datetime | None:
+    if not s:
+        return None
+    try:
+        return dateutil_parse(s)
+    except (ParserError, ValueError, OverflowError):
+        return None
+
+
+def _proximity_score(q: datetime, e: datetime) -> float:
+    """Gaussian proximity: same day=1.0, 30d=0.61, 90d=0.11."""
+    dist = abs((q - e).total_seconds()) / 86400.0
+    if dist > _MAX_PROXIMITY_DAYS:
+        return 0.0
+    return math.exp(-(dist * dist) / (2.0 * 30.0 * 30.0))
+
+
+class TemporalChannel:
+    """Date-aware retrieval using the 3-date temporal model."""
+
+    def __init__(self, db: DatabaseManager) -> None:
+        self._db = db
+
+    def search(self, query: str, profile_id: str, top_k: int = 30) -> list[tuple[str, float]]:
+        """Search for temporally relevant facts.
+
+        Two strategies:
+        1. Date proximity: scores events by date closeness to query date.
+        2. Entity-temporal: filters events by entity name in query,
+           returns ALL their temporal facts (metadata-first approach).
+
+        Returns empty only when query has no temporal signal AND no
+        entity-temporal matches.
+        """
+        parser = TemporalParser()
+        dates = parser.extract_dates_from_text(query)
+        query_dt = _parse_iso(dates.get("referenced_date"))
+        if query_dt is None:
+            query_dt = self._try_parse(query)
+
+        # Strategy 1: Entity-temporal metadata search
+        # "When did Alice...?" → find all temporal events for Alice
+        entity_results = self._entity_temporal_search(query, profile_id)
+
+        # Strategy 2: Date proximity search
+        if query_dt is None and not entity_results:
+            return []
+
+        events = self._load_events(profile_id)
+        scored: dict[str, float] = {}
+
+        # Include entity-temporal results with high base score
+        for fid, score in entity_results:
+            scored[fid] = max(scored.get(fid, 0.0), score)
+
+        if query_dt is not None:
+            for ev in events:
+                best = 0.0
+                ref = _parse_iso(ev.get("referenced_date"))
+                if ref is not None:
+                    best = max(best, _proximity_score(query_dt, ref))
+
+                obs = _parse_iso(ev.get("observation_date"))
+                if obs is not None:
+                    best = max(best, _proximity_score(query_dt, obs) * 0.8)
+
+                i_start = _parse_iso(ev.get("interval_start"))
+                i_end = _parse_iso(ev.get("interval_end"))
+                if i_start and i_end:
+                    if i_start <= query_dt <= i_end:
+                        best = max(best, 1.0)
+                    else:
+                        best = max(best, max(
+                            _proximity_score(query_dt, i_start),
+                            _proximity_score(query_dt, i_end),
+                        ) * 0.9)
+
+                if best > 0.0:
+                    fid = ev["fact_id"]
+                    scored[fid] = max(scored.get(fid, 0.0), best)
+
+        results = sorted(scored.items(), key=lambda x: x[1], reverse=True)
+        return results[:top_k]
+
+    def _entity_temporal_search(
+        self, query: str, profile_id: str,
+    ) -> list[tuple[str, float]]:
+        """Metadata-first: find temporal events for entities mentioned in query.
+
+        "When did Alice do X?" → SQL filter by entity_id for Alice → return
+        all temporal facts about Alice. High precision for entity+time queries.
+        """
+        import re
+        _PROPER_RE = re.compile(r"\b([A-Z][a-z]+)\b")
+        names = [m.group(1) for m in _PROPER_RE.finditer(query)]
+        if not names:
+            return []
+
+        results: list[tuple[str, float]] = []
+        seen: set[str] = set()
+
+        for name in names[:3]:  # Limit to first 3 entity mentions
+            # Look up entity ID
+            entity = self._db.get_entity_by_name(name, profile_id)
+            if entity is None:
+                continue
+
+            # Find all temporal events for this entity
+            rows = self._db.execute(
+                "SELECT fact_id FROM temporal_events "
+                "WHERE profile_id = ? AND entity_id = ?",
+                (profile_id, entity.entity_id),
+            )
+            for row in rows:
+                fid = dict(row)["fact_id"]
+                if fid not in seen:
+                    seen.add(fid)
+                    results.append((fid, 0.85))  # High base score for entity-temporal
+
+        return results
+
+    def _load_events(self, profile_id: str) -> list[dict]:
+        rows = self._db.execute(
+            "SELECT fact_id, observation_date, referenced_date, "
+            "interval_start, interval_end "
+            "FROM temporal_events WHERE profile_id = ?",
+            (profile_id,),
+        )
+        return [dict(r) for r in rows]
+
+    @staticmethod
+    def _try_parse(text: str) -> datetime | None:
+        """Fuzzy date parse with safety guards.
+
+        dateutil fuzzy=True is exponential on long non-date text.
+        Guard: only attempt on short strings (< 60 chars) that contain
+        at least one digit (dates always have numbers).
+        """
+        if len(text) > 60 or not any(c.isdigit() for c in text):
+            return None
+        try:
+            return dateutil_parse(text, fuzzy=True)
+        except (ParserError, ValueError, OverflowError):
+            return None

package/src/superlocalmemory/server/__init__.py

@@ -0,0 +1 @@
+# Server package — API and UI servers