superlocalmemory 2.8.5 → 3.0.0

package/src/superlocalmemory/math/langevin.py

@@ -0,0 +1,398 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under the MIT License - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""Riemannian Langevin dynamics with persistence for memory lifecycle.
+
+Evolves memory positions via a discretised Langevin SDE driven by an
+access-aware potential function.  Frequently accessed memories are
+confined near the origin (ACTIVE, high retrieval weight) by a strong
+potential, while neglected memories diffuse toward the boundary
+(ARCHIVED, weight approaches 0) as their confinement weakens.
+
+Core SDE (Girolami & Calderhead 2011):
+
+    xi_{t+1} = xi_t - g^{-1}(xi_t) * grad U(xi_t) * dt
+             + sqrt(2 * T * dt) * g^{-1/2}(xi_t) * eta
+
+Potential function:
+
+    U(xi) = alpha * ||xi||^2
+           - beta  * log(access_count + 1)
+           + gamma * age_days
+           - delta * importance
+
+    - Accessed memories:  beta term lowers potential  -> drift to origin
+    - Old / unused:       gamma term raises potential -> drift to boundary
+    - Important memories: delta term lowers potential -> retain near origin
+
+V1 bugs fixed in Innovation Wave 4:
+    1. Positions were computed per-recall then DISCARDED.  Now ``step()``
+       and ``batch_step()`` return new positions for the caller to persist.
+    2. Weight range was [0.7, 1.0] --- too narrow to change rankings.
+       Now [0.0, 1.0] so archived memories can be effectively suppressed.
+
+References:
+    Girolami M & Calderhead B (2011). Riemann manifold Langevin and
+        Hamiltonian Monte Carlo methods. JRSS-B, 73(2), 123--214.
+    Roberts G O & Tweedie R L (1996). Exponential convergence of Langevin
+        distributions. Bernoulli, 2(4), 341--363.
+    Xifara T, Sherlock C, Livingstone S, Byrne S & Girolami M (2014).
+        Langevin diffusions and the MALA algorithm. Statistics &
+        Probability Letters, 91, 59--67.
+
+Part of Qualixar | Author: Varun Pratap Bhardwaj
+License: MIT
+"""
+
+from __future__ import annotations
+
+import math
+from typing import Any
+
+import numpy as np
+
+from superlocalmemory.storage.models import MemoryLifecycle
+
+# ---------------------------------------------------------------------------
+# Lifecycle zone boundaries (radius thresholds on the unit ball)
+# ---------------------------------------------------------------------------
+
+_RADIUS_ACTIVE: float = 0.3    # [0, 0.3)    -> ACTIVE
+_RADIUS_WARM: float = 0.55     # [0.3, 0.55) -> WARM
+_RADIUS_COLD: float = 0.8      # [0.55, 0.8) -> COLD
+                                # [0.8, 1.0)  -> ARCHIVED
+
+# Potential coefficients (defaults)
+# Alpha must be > 0.5*T*(d-2)/lam_inv ≈ 1.8 at T=0.3 for confinement
+# to counter the Riemannian curvature correction near the origin.
+_ALPHA: float = 3.0   # Base confinement (strong enough to counter correction)
+_BETA: float = 0.8    # Access STRENGTHENS confinement (keeps active facts near origin)
+_GAMMA: float = 0.005 # Age WEAKENS confinement (old facts drift to boundary)
+_DELTA: float = 0.5   # Importance STRENGTHENS confinement
+
+# Safety
+_MAX_NORM: float = 0.99   # Clamp radius < 1 (open ball)
+_EPS: float = 1e-8
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+class LangevinDynamics:
+    """Riemannian Langevin dynamics with persistence for memory lifecycle.
+
+    Positions live on the open unit ball B^d.  The potential function
+    encodes access frequency, age, and importance as competing forces.
+    The lifecycle state (ACTIVE / WARM / COLD / ARCHIVED) is determined
+    purely by radial position.
+
+    Attributes:
+        dt:           Time-step size for Euler-Maruyama integration.
+        temperature:  Boltzmann temperature controlling diffusion spread.
+        weight_range: (min_weight, max_weight) for lifecycle weighting.
+        dim:          Dimensionality of the position space.
+    """
+
+    __slots__ = (
+        "dt", "temperature", "weight_range", "dim",
+        "_alpha", "_beta", "_gamma", "_delta",
+    )
+
+    def __init__(
+        self,
+        dt: float = 0.01,
+        temperature: float = 1.0,
+        weight_range: tuple[float, float] = (0.0, 1.0),
+        dim: int = 8,
+    ) -> None:
+        if dt <= 0:
+            raise ValueError(f"dt must be positive, got {dt}")
+        if temperature <= 0:
+            raise ValueError(f"temperature must be positive, got {temperature}")
+        if weight_range[0] > weight_range[1]:
+            raise ValueError(
+                f"weight_range min > max: {weight_range[0]} > {weight_range[1]}"
+            )
+
+        self.dt = dt
+        self.temperature = temperature
+        self.weight_range = weight_range
+        self.dim = dim
+
+        # Potential coefficients
+        self._alpha = _ALPHA
+        self._beta = _BETA
+        self._gamma = _GAMMA
+        self._delta = _DELTA
+
+    # ------------------------------------------------------------------
+    # Single-fact step
+    # ------------------------------------------------------------------
+
+    def step(
+        self,
+        position: list[float],
+        access_count: int,
+        age_days: float,
+        importance: float,
+        seed: int | None = None,
+    ) -> tuple[list[float], float]:
+        """One Euler-Maruyama step of the Langevin SDE.
+
+        Computes the gradient of the potential at the current position,
+        applies drift + stochastic diffusion, and returns the new position
+        plus the resulting lifecycle weight.
+
+        The caller is responsible for PERSISTING the returned position
+        back to the database.  This is the key V1 fix: positions must
+        evolve across recalls, not be recomputed from scratch each time.
+
+        Args:
+            position:     Current position on the unit ball, length ``dim``.
+            access_count: Total number of times this fact has been accessed.
+            age_days:     Days since the fact was first stored.
+            importance:   Fact importance score in [0, 1].
+            seed:         Optional RNG seed for reproducible diffusion.
+
+        Returns:
+            (new_position, lifecycle_weight) where new_position is a list
+            of floats inside the unit ball and lifecycle_weight is in
+            [weight_range[0], weight_range[1]].
+        """
+        xi = np.asarray(position, dtype=np.float64)
+        if xi.shape[0] != self.dim:
+            # Graceful resize: pad or truncate
+            xi = _resize_position(xi, self.dim)
+
+        # --- Gradient of potential ---
+        grad = self._potential_gradient(xi, access_count, age_days, importance)
+
+        # --- Conformal factor on Poincare ball: lambda = 2 / (1 - ||x||^2) ---
+        norm_sq = float(np.dot(xi, xi))
+        norm_sq = min(norm_sq, _MAX_NORM ** 2)
+        lam = 2.0 / (1.0 - norm_sq + _EPS)
+        lam_inv = 1.0 / lam
+
+        # --- Drift: -lambda^{-2} * grad_U * dt (Eq. 5 term 1) ---
+        drift = -(lam_inv ** 2) * grad * self.dt
+
+        # --- Curvature correction: 0.5 * T * (d-2) * lambda^{-1} * xi * dt (Eq. 5 term 3) ---
+        correction = 0.5 * self.temperature * (self.dim - 2) * lam_inv * xi * self.dt
+
+        # --- Diffusion: sqrt(2T * dt) * lambda^{-1} * noise (Eq. 5 term 2) ---
+        rng = np.random.default_rng(seed)
+        noise = rng.standard_normal(self.dim)
+        diffusion = math.sqrt(2.0 * self.temperature * self.dt) * lam_inv * noise
+
+        # --- Full Euler-Maruyama update (Girolami & Calderhead 2011) ---
+        new_xi = xi + drift + correction + diffusion
+
+        # --- Project back into the open ball ---
+        new_xi = _project_to_ball(new_xi)
+
+        weight = self.compute_lifecycle_weight(new_xi.tolist())
+        return new_xi.tolist(), weight
+
+    # ------------------------------------------------------------------
+    # Lifecycle weight from position
+    # ------------------------------------------------------------------
+
+    def compute_lifecycle_weight(self, position: list[float]) -> float:
+        """Compute retrieval weight from radial position.
+
+        Weight decreases linearly from max to min as radius moves from
+        origin to boundary.  This ensures archived memories (near boundary)
+        are strongly suppressed in retrieval rankings.
+
+        Args:
+            position: Current position, length ``dim``.
+
+        Returns:
+            Weight in [weight_range[0], weight_range[1]].
+        """
+        xi = np.asarray(position, dtype=np.float64)
+        radius = float(np.linalg.norm(xi))
+        radius = min(radius, _MAX_NORM)
+
+        # Linear interpolation: radius 0 -> max weight, radius 1 -> min weight
+        lo, hi = self.weight_range
+        weight = hi - (hi - lo) * (radius / _MAX_NORM)
+        return max(lo, min(hi, weight))
+
+    # ------------------------------------------------------------------
+    # Lifecycle state classification
+    # ------------------------------------------------------------------
+
+    def get_lifecycle_state(self, weight: float) -> MemoryLifecycle:
+        """Classify lifecycle state from weight.
+
+        Uses the weight (derived from radius) rather than raw radius,
+        so the caller doesn't need to know zone boundaries.
+
+        Args:
+            weight: Lifecycle weight in [0, 1].
+
+        Returns:
+            MemoryLifecycle enum value.
+        """
+        # Map weight back to approximate radius for zone classification
+        lo, hi = self.weight_range
+        span = hi - lo
+        if span < _EPS:
+            return MemoryLifecycle.ACTIVE
+
+        # radius_approx: weight=hi -> radius=0, weight=lo -> radius=MAX_NORM
+        radius_approx = (hi - weight) / span * _MAX_NORM
+
+        if radius_approx < _RADIUS_ACTIVE:
+            return MemoryLifecycle.ACTIVE
+        if radius_approx < _RADIUS_WARM:
+            return MemoryLifecycle.WARM
+        if radius_approx < _RADIUS_COLD:
+            return MemoryLifecycle.COLD
+        return MemoryLifecycle.ARCHIVED
+
+    # ------------------------------------------------------------------
+    # Batch step (background maintenance)
+    # ------------------------------------------------------------------
+
+    def batch_step(
+        self,
+        facts: list[dict[str, Any]],
+        seed: int | None = None,
+    ) -> list[dict[str, Any]]:
+        """Batch Langevin step for background maintenance.
+
+        Evolves all fact positions in a single pass.  Designed to be
+        called periodically (e.g. every N hours) to let the memory store
+        self-organise over time.
+
+        Each input dict must contain:
+            - ``fact_id``:      str
+            - ``position``:     list[float]
+            - ``access_count``: int
+            - ``age_days``:     float
+            - ``importance``:   float
+
+        Args:
+            facts: List of fact dicts with position and metadata.
+            seed:  Optional base seed (incremented per fact for variety).
+
+        Returns:
+            List of dicts with ``fact_id``, ``position`` (updated),
+            ``weight``, and ``lifecycle`` fields.
+        """
+        results: list[dict[str, Any]] = []
+        base_seed = seed
+
+        for i, fact in enumerate(facts):
+            fact_seed = (base_seed + i) if base_seed is not None else None
+
+            new_pos, weight = self.step(
+                position=fact["position"],
+                access_count=fact["access_count"],
+                age_days=fact["age_days"],
+                importance=fact["importance"],
+                seed=fact_seed,
+            )
+
+            results.append({
+                "fact_id": fact["fact_id"],
+                "position": new_pos,
+                "weight": weight,
+                "lifecycle": self.get_lifecycle_state(weight).value,
+            })
+
+        return results
+
+    # ------------------------------------------------------------------
+    # Potential gradient (private)
+    # ------------------------------------------------------------------
+
+    def _potential_gradient(
+        self,
+        xi: np.ndarray,
+        access_count: int,
+        age_days: float,
+        importance: float,
+    ) -> np.ndarray:
+        """Metadata-modulated confinement gradient .
+
+        Potential:
+            U(xi) = effective_alpha * ||xi||^2
+
+        Where effective_alpha is modulated by metadata:
+            effective_alpha = alpha
+                + beta * log(access+1)/10   (frequent access → stronger confinement → ACTIVE)
+                - gamma * min(age, 365)/365  (aging → weaker confinement → drifts to ARCHIVED)
+                + delta * importance          (important → stronger confinement → stays ACTIVE)
+
+        Gradient:
+            grad U = 2 * effective_alpha * xi
+
+        This ensures metadata drives correct lifecycle dynamics:
+        - Frequently accessed facts stay near origin (ACTIVE, high weight)
+        - Old unused facts drift toward boundary (ARCHIVED, low weight)
+        - Important facts resist archival (persistent)
+
+        Args:
+            xi:           Position array.
+            access_count: Total access count.
+            age_days:     Days since creation.
+            importance:   Importance score [0, 1].
+
+        Returns:
+            Gradient vector, same shape as xi.
+        """
+        import math as _math
+        effective_alpha = (
+            self._alpha
+            + self._beta * _math.log(access_count + 1) / 10.0
+            - self._gamma * min(age_days, 365.0) / 365.0
+            + self._delta * importance
+        )
+        effective_alpha = max(0.1, effective_alpha)
+        return 2.0 * effective_alpha * xi
+
+
+# ---------------------------------------------------------------------------
+# Helpers (module-private)
+# ---------------------------------------------------------------------------
+
+def _project_to_ball(xi: np.ndarray) -> np.ndarray:
+    """Project point back into the open unit ball.
+
+    If ||xi|| >= MAX_NORM, rescale to sit at MAX_NORM.
+
+    Args:
+        xi: Position array.
+
+    Returns:
+        Projected position with ||result|| < 1.
+    """
+    norm = float(np.linalg.norm(xi))
+    if norm >= _MAX_NORM:
+        return xi * (_MAX_NORM / (norm + _EPS))
+    return xi
+
+
+def _resize_position(xi: np.ndarray, target_dim: int) -> np.ndarray:
+    """Resize a position vector to target dimensionality.
+
+    Pads with zeros or truncates as needed.
+
+    Args:
+        xi:         Source position.
+        target_dim: Desired dimensionality.
+
+    Returns:
+        Resized position array.
+    """
+    current = xi.shape[0]
+    if current == target_dim:
+        return xi
+    if current < target_dim:
+        return np.pad(xi, (0, target_dim - current), constant_values=0.0)
+    return xi[:target_dim]

package/src/superlocalmemory/math/sheaf.py

@@ -0,0 +1,257 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under the MIT License - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""Sheaf cohomology for contradiction detection at ENCODING time.
+
+V1 applied sheaf at RECALL (HARMFUL — penalized diverse multi-hop results).
+V2 runs at ENCODING ONLY, checking new facts against graph-connected facts.
+
+Coboundary: delta_0 f(e) = R_b emb_b - R_a emb_a
+Restriction maps by edge type: ENTITY=I, TEMPORAL=s*I (s<1), SEMANTIC=I.
+Severity = ||delta_0 f(e)|| / (||R_a emb_a|| + ||R_b emb_b||).
+
+Refs: Curry 2014 (arXiv:1303.3255), Hansen & Ghrist 2019, Robinson 2020.
+
+Part of Qualixar | Author: Varun Pratap Bhardwaj
+License: MIT
+"""
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+import numpy as np
+
+if TYPE_CHECKING:
+    from superlocalmemory.storage.database import DatabaseManager
+    from superlocalmemory.storage.models import AtomicFact
+
+logger = logging.getLogger(__name__)
+
+_EPS = 1e-12
+
+# Temporal tolerance: fraction of coboundary norm below which temporal
+# edges are NOT flagged as contradictions (legitimate state changes).
+TEMPORAL_TOLERANCE: float = 0.15
+
+
+# ---------------------------------------------------------------------------
+# Result type
+# ---------------------------------------------------------------------------
+
+@dataclass(frozen=True)
+class ContradictionResult:
+    """Detected contradiction between two facts."""
+
+    fact_id_a: str   # New fact being stored
+    fact_id_b: str   # Existing fact it conflicts with
+    severity: float  # Normalized disagreement [0.0, 1.0]
+    edge_type: str   # Graph edge type (entity/temporal/semantic/causal)
+    description: str
+
+
+# ---------------------------------------------------------------------------
+# Restriction maps — non-trivial (V1 used identity everywhere)
+# ---------------------------------------------------------------------------
+
+def _restriction_for_edge_type(
+    edge_type: str,
+    dim: int,
+    emb_a: np.ndarray | None = None,
+    emb_b: np.ndarray | None = None,
+) -> np.ndarray:
+    """Edge-type-specific restriction map.
+
+    Entity edges use non-trivial maps that AMPLIFY the disagreement
+    along the axis of maximum difference between embeddings:
+        R = 0.7*I + 0.3 * outer(diff, diff) / ||diff||^2
+
+    This projects contradictions onto the most discriminative subspace,
+    making the coboundary norm more sensitive to real disagreements.
+
+    Temporal edges use identity restriction (scalar multiples cancel
+    in the normalized coboundary). Temporal tolerance is applied at
+    the caller level via TEMPORAL_TOLERANCE threshold.
+    """
+    if edge_type == "entity" and emb_a is not None and emb_b is not None:
+        diff = emb_a - emb_b
+        norm_sq = max(float(np.dot(diff, diff)), 1e-12)
+        return 0.7 * np.eye(dim) + 0.3 * np.outer(diff, diff) / norm_sq
+    return np.eye(dim)
+
+
+def edge_residual(
+    emb_a: np.ndarray, emb_b: np.ndarray,
+    R_a: np.ndarray, R_b: np.ndarray,
+) -> np.ndarray:
+    """Coboundary residual: delta_0 f(e) = R_b emb_b - R_a emb_a."""
+    return R_b @ emb_b - R_a @ emb_a
+
+
+def coboundary_norm(
+    emb_a: np.ndarray, emb_b: np.ndarray,
+    R_a: np.ndarray, R_b: np.ndarray,
+) -> float:
+    """Normalized coboundary: ||delta|| / (||R_a emb_a|| + ||R_b emb_b||).
+
+    Returns [0, ~2]. Near 0 = agree; near 1+ = strong disagreement.
+    """
+    residual = edge_residual(emb_a, emb_b, R_a, R_b)
+    res_norm = float(np.linalg.norm(residual))
+    denom = float(np.linalg.norm(R_a @ emb_a) + np.linalg.norm(R_b @ emb_b) + _EPS)
+    return res_norm / denom
+
+
+class SheafConsistencyChecker:
+    """Detect contradictions at ENCODING time (not retrieval).
+
+    V1 differences: uses real graph edges (not all-pairs), non-trivial
+    restriction maps, and returns results for the pipeline to act on.
+    """
+
+    def __init__(
+        self,
+        db: DatabaseManager,
+        contradiction_threshold: float = 0.7,
+    ) -> None:
+        self._db = db
+        self._threshold = max(0.0, min(2.0, contradiction_threshold))
+
+    # -- Public API ---------------------------------------------------------
+
+    def check_consistency(
+        self,
+        new_fact: AtomicFact,
+        profile_id: str,
+    ) -> list[ContradictionResult]:
+        """Check new fact against graph-connected existing facts."""
+        if new_fact.embedding is None:
+            return []
+        if not new_fact.canonical_entities:
+            return []
+
+        emb_a = np.asarray(new_fact.embedding, dtype=np.float64)
+        dim = emb_a.shape[0]
+        if dim == 0:
+            return []
+
+        contradictions: list[ContradictionResult] = []
+        checked_pairs: set[str] = set()
+
+        # Get all graph edges touching this fact
+        edges = self._db.get_edges_for_node(new_fact.fact_id, profile_id)
+
+        for edge in edges:
+            # Determine the OTHER fact in this edge
+            other_id = (
+                edge.target_id
+                if edge.source_id == new_fact.fact_id
+                else edge.source_id
+            )
+
+            # Skip already-checked pairs and skip contradiction/supersedes edges
+            if other_id in checked_pairs:
+                continue
+            if edge.edge_type.value in ("contradiction", "supersedes"):
+                continue
+            checked_pairs.add(other_id)
+
+            # Look up the other fact's embedding
+            other_emb = self._get_fact_embedding(other_id)
+            if other_emb is None or other_emb.shape[0] != dim:
+                continue
+
+            # Compute coboundary with edge-type-specific restriction map
+            edge_type_str = edge.edge_type.value
+            R = _restriction_for_edge_type(edge_type_str, dim, emb_a, other_emb)
+            severity = coboundary_norm(emb_a, other_emb, R, R)
+
+            if severity > self._threshold:
+                contradictions.append(ContradictionResult(
+                    fact_id_a=new_fact.fact_id,
+                    fact_id_b=other_id,
+                    severity=min(severity, 1.0),
+                    edge_type=edge_type_str,
+                    description=(
+                        f"Sheaf coboundary {severity:.3f} > {self._threshold:.2f} "
+                        f"along {edge_type_str} edge"
+                    ),
+                ))
+
+        if contradictions:
+            logger.info(
+                "Sheaf: %d contradiction(s) for fact %s",
+                len(contradictions), new_fact.fact_id,
+            )
+        return contradictions
+
+    def detect_contradictions_batch(
+        self,
+        facts: list[AtomicFact],
+        profile_id: str,
+    ) -> list[ContradictionResult]:
+        """Pairwise check within entity groups (for batch imports)."""
+        # Group facts by canonical entity
+        entity_groups: dict[str, list[AtomicFact]] = {}
+        for fact in facts:
+            if fact.embedding is None:
+                continue
+            for eid in fact.canonical_entities:
+                entity_groups.setdefault(eid, []).append(fact)
+
+        contradictions: list[ContradictionResult] = []
+        checked_pairs: set[tuple[str, str]] = set()
+
+        for _entity_id, group in entity_groups.items():
+            for i in range(len(group)):
+                for j in range(i + 1, len(group)):
+                    fa, fb = group[i], group[j]
+                    pair = (min(fa.fact_id, fb.fact_id), max(fa.fact_id, fb.fact_id))
+                    if pair in checked_pairs:
+                        continue
+                    checked_pairs.add(pair)
+
+                    emb_a = np.asarray(fa.embedding, dtype=np.float64)
+                    emb_b = np.asarray(fb.embedding, dtype=np.float64)
+                    if emb_a.shape[0] != emb_b.shape[0] or emb_a.shape[0] == 0:
+                        continue
+
+                    dim = emb_a.shape[0]
+                    R = np.eye(dim)
+                    severity = coboundary_norm(emb_a, emb_b, R, R)
+
+                    if severity > self._threshold:
+                        contradictions.append(ContradictionResult(
+                            fact_id_a=fa.fact_id,
+                            fact_id_b=fb.fact_id,
+                            severity=min(severity, 1.0),
+                            edge_type="entity",
+                            description=(
+                                f"Batch sheaf coboundary {severity:.3f} > "
+                                f"{self._threshold:.2f} (shared entity group)"
+                            ),
+                        ))
+
+        return contradictions
+
+    # -- Internal -----------------------------------------------------------
+
+    def _get_fact_embedding(self, fact_id: str) -> np.ndarray | None:
+        """Load a fact's embedding from the database."""
+        import json
+        rows = self._db.execute(
+            "SELECT embedding FROM atomic_facts WHERE fact_id = ?",
+            (fact_id,),
+        )
+        if not rows:
+            return None
+        raw = dict(rows[0]).get("embedding")
+        if raw is None or raw == "":
+            return None
+        try:
+            data = json.loads(raw) if isinstance(raw, str) else raw
+            return np.asarray(data, dtype=np.float64)
+        except (json.JSONDecodeError, TypeError, ValueError):
+            return None