superlocalmemory 3.2.2 → 3.3.0

package/src/superlocalmemory/dynamics/activation_guided_quantization.py

@@ -0,0 +1,374 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under the MIT License - see LICENSE file
+# Part of SuperLocalMemory V3
+
+"""Spreading Activation-Guided Quantization (SAGQ) engine.
+
+Novel contribution: uses graph centrality (PageRank + degree + SA frequency)
+to allocate embedding precision. Well-connected memories keep higher precision
+because they serve as hubs for spreading activation retrieval.
+
+Core formula:
+    centrality(i) = w_pr * pr_norm + w_deg * deg_norm + w_sa * sa_freq_norm
+    sagq_bw = b_min + (b_max - b_min) * centrality, ceil-snapped to valid set
+
+Conflict resolution with Phase A EAP:
+    final_bw = max(eap_bw, sagq_bw) -- safety first, never over-quantize
+
+HR-02: Conflict resolution is ALWAYS max().
+HR-03: Centrality scores always in [0.0, 1.0].
+HR-04: Bit-width always from valid_bit_widths.
+HR-05: All SQL uses parameterized queries.
+HR-07: No-op when config.enabled=False.
+
+Part of Qualixar | Author: Varun Pratap Bhardwaj
+License: MIT
+"""
+
+from __future__ import annotations
+
+import logging
+import math
+from dataclasses import dataclass
+from typing import Any, Callable, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from superlocalmemory.storage.database import DatabaseManager
+    from superlocalmemory.core.config import SAGQConfig
+
+logger = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# Data classes (frozen -- all immutable, Rule 10)
+# ---------------------------------------------------------------------------
+
+
+@dataclass(frozen=True)
+class CentralityScore:
+    """Computed centrality for a single memory node."""
+
+    fact_id: str
+    pagerank_norm: float       # pr_norm(i) in [0, 1]
+    degree_norm: float         # deg_norm(i) in [0, 1]
+    sa_freq_norm: float        # sa_freq(i) in [0, 1]
+    combined_centrality: float  # weighted sum in [0, 1]
+
+
+@dataclass(frozen=True)
+class SAGQPrecision:
+    """SAGQ precision recommendation for a single memory."""
+
+    fact_id: str
+    centrality: float    # combined centrality in [0, 1]
+    sagq_bit_width: int  # recommended bit-width from SAGQ signal
+    eap_bit_width: int   # recommended bit-width from EAP signal (Phase A)
+    final_bit_width: int  # max(sagq_bit_width, eap_bit_width)
+    current_bit_width: int  # current bit-width from embedding_metadata
+    action: str          # "upgrade" | "downgrade" | "skip"
+
+
+# ---------------------------------------------------------------------------
+# ActivationGuidedQuantizer
+# ---------------------------------------------------------------------------
+
+
+class ActivationGuidedQuantizer:
+    """SAGQ engine: maps graph centrality to embedding precision.
+
+    Reads from fact_importance (PageRank, degree) and activation_cache
+    (spreading activation frequency) to compute centrality per memory.
+    Maps centrality to bit-width via linear interpolation + ceiling snap.
+    """
+
+    def __init__(self, db: Any, config: Any) -> None:
+        """Initialize SAGQ quantizer. No side effects."""
+        self._db = db
+        self._config = config
+
+    def compute_centrality_batch(
+        self, profile_id: str,
+    ) -> list[CentralityScore]:
+        """Compute centrality for all facts in a profile.
+
+        Returns list of CentralityScore, each with combined_centrality in [0, 1].
+        Returns empty list if disabled or no data.
+        """
+        if not self._config.enabled:
+            return []
+
+        # Step 1: Query fact_importance (Q1)
+        try:
+            rows = self._db.execute(
+                "SELECT fact_id, pagerank_score, degree_centrality "
+                "FROM fact_importance "
+                "WHERE profile_id = ?",
+                (profile_id,),
+            )
+        except Exception as exc:
+            logger.warning("SAGQ: fact_importance query failed: %s", exc)
+            return []
+
+        if not rows:
+            logger.info("SAGQ: no importance data yet for profile %s, skipping", profile_id)
+            return []
+
+        # Step 3: Compute max values for normalization (avoid division by zero)
+        fact_data = [(dict(r)["fact_id"], dict(r)["pagerank_score"], dict(r)["degree_centrality"]) for r in rows]
+
+        max_pr = max((pr for _, pr, _ in fact_data), default=0.0)
+        max_deg = max((deg for _, _, deg in fact_data), default=0.0)
+        if max_pr == 0.0:
+            max_pr = 1.0
+        if max_deg == 0.0:
+            max_deg = 1.0
+
+        # Step 4: Query activation_cache for SA frequency (Q2)
+        sa_window = f"-{self._config.sa_frequency_window_days} days"
+        try:
+            sa_rows = self._db.execute(
+                "SELECT node_id, COUNT(*) as activation_count "
+                "FROM activation_cache "
+                "WHERE profile_id = ? "
+                "AND created_at > datetime('now', ?) "
+                "GROUP BY node_id",
+                (profile_id, sa_window),
+            )
+        except Exception as exc:
+            logger.debug("SAGQ: activation_cache query failed: %s", exc)
+            sa_rows = []
+
+        # Step 5: Build SA frequency map
+        sa_freq_map: dict[str, int] = {}
+        for sa_row in sa_rows:
+            d = dict(sa_row)
+            sa_freq_map[d["node_id"]] = int(d["activation_count"])
+
+        # Step 6: max SA frequency
+        max_sa = max(sa_freq_map.values()) if sa_freq_map else 1
+
+        # Step 7: Compute centrality for each fact
+        cfg = self._config
+        result: list[CentralityScore] = []
+        for fact_id, pr_score, deg_score in fact_data:
+            pr_norm = pr_score / max_pr
+            deg_norm = deg_score / max_deg
+            sa_freq = sa_freq_map.get(fact_id, 0)
+            sa_freq_norm = sa_freq / max_sa
+
+            combined = (
+                cfg.w_pagerank * pr_norm
+                + cfg.w_degree * deg_norm
+                + cfg.w_sa_freq * sa_freq_norm
+            )
+
+            # NaN safety (HR-03)
+            if math.isnan(combined):
+                logger.warning("SAGQ: NaN centrality for %s, defaulting to 0.0", fact_id)
+                combined = 0.0
+
+            # Clamp to [0.0, 1.0]
+            combined = max(0.0, min(1.0, combined))
+
+            result.append(CentralityScore(
+                fact_id=fact_id,
+                pagerank_norm=pr_norm,
+                degree_norm=deg_norm,
+                sa_freq_norm=sa_freq_norm,
+                combined_centrality=combined,
+            ))
+
+        return result
+
+    def centrality_to_bit_width(self, centrality: float) -> int:
+        """Map centrality score to SAGQ bit-width.
+
+        Linear interpolation from [0,1] to [b_min, b_max], then ceiling-snap
+        to nearest valid bit-width. SAGQ is a preservation signal -- always
+        round UP (Decision D1).
+
+        Returns one of valid_bit_widths.
+        """
+        cfg = self._config
+
+        # Clamp input
+        centrality = max(0.0, min(1.0, centrality))
+
+        # Linear mapping
+        raw_bw = cfg.b_min + (cfg.b_max - cfg.b_min) * centrality
+
+        # Ceiling snap to nearest valid bit-width (smallest >= raw_bw)
+        for vbw in cfg.valid_bit_widths:
+            if vbw >= raw_bw:
+                return vbw
+
+        # raw_bw exceeds all valid values -- cap at maximum
+        return cfg.valid_bit_widths[-1]
+
+    def compute_sagq_precision_batch(
+        self,
+        profile_id: str,
+        eap_precision_fn: Callable[[str], int],
+    ) -> list[SAGQPrecision]:
+        """Compute combined SAGQ + EAP precision for all facts in a profile.
+
+        Args:
+            profile_id: Profile to process.
+            eap_precision_fn: Callable(fact_id) -> EAP bit-width (from Phase A).
+
+        Returns:
+            List of SAGQPrecision with action ("upgrade"/"downgrade"/"skip").
+        """
+        if not self._config.enabled:
+            return []
+
+        # Step 1: Compute centrality
+        centrality_scores = self.compute_centrality_batch(profile_id)
+        if not centrality_scores:
+            return []
+
+        # Step 3: Batch-fetch current bit-widths (Q3)
+        try:
+            bw_rows = self._db.execute(
+                "SELECT fact_id, COALESCE(bit_width, 32) as bit_width "
+                "FROM embedding_metadata "
+                "WHERE profile_id = ?",
+                (profile_id,),
+            )
+        except Exception as exc:
+            logger.warning("SAGQ: embedding_metadata query failed: %s", exc)
+            bw_rows = []
+
+        current_bw_map: dict[str, int] = {}
+        for row in bw_rows:
+            d = dict(row)
+            current_bw_map[d["fact_id"]] = int(d["bit_width"])
+
+        # Step 5: Compute precision for each fact
+        result: list[SAGQPrecision] = []
+        for cs in centrality_scores:
+            # SAGQ signal
+            sagq_bw = self.centrality_to_bit_width(cs.combined_centrality)
+
+            # EAP signal
+            eap_bw = eap_precision_fn(cs.fact_id)
+
+            # HR-02: Conflict resolution -- ALWAYS max()
+            final_bw = max(sagq_bw, eap_bw)
+
+            # Current bit-width
+            current_bw = current_bw_map.get(cs.fact_id, 32)
+
+            # Determine action
+            if final_bw < current_bw:
+                action = "downgrade"
+            elif final_bw > current_bw:
+                action = "upgrade"
+            else:
+                action = "skip"
+
+            result.append(SAGQPrecision(
+                fact_id=cs.fact_id,
+                centrality=cs.combined_centrality,
+                sagq_bit_width=sagq_bw,
+                eap_bit_width=eap_bw,
+                final_bit_width=final_bw,
+                current_bit_width=current_bw,
+                action=action,
+            ))
+
+        return result
+
+    def get_centrality_for_fact(
+        self, fact_id: str, profile_id: str,
+    ) -> float:
+        """Get centrality for a single fact. Returns 0.0 if not found.
+
+        Used by Phase E (CCQ) for centrality-aware consolidation.
+        """
+        if not self._config.enabled:
+            return 0.0
+
+        # Step 1: Query single fact importance (Q11)
+        try:
+            rows = self._db.execute(
+                "SELECT pagerank_score, degree_centrality "
+                "FROM fact_importance "
+                "WHERE fact_id = ? AND profile_id = ?",
+                (fact_id, profile_id),
+            )
+        except Exception as exc:
+            logger.debug("SAGQ: single fact query failed: %s", exc)
+            return 0.0
+
+        if not rows:
+            return 0.0
+
+        d = dict(rows[0])
+        pr_score = float(d["pagerank_score"])
+        deg_score = float(d["degree_centrality"])
+
+        # Step 3: Normalization (Q4)
+        try:
+            max_rows = self._db.execute(
+                "SELECT "
+                "  COALESCE(MAX(pagerank_score), 0.0) as max_pr, "
+                "  COALESCE(MAX(degree_centrality), 0.0) as max_deg "
+                "FROM fact_importance "
+                "WHERE profile_id = ?",
+                (profile_id,),
+            )
+        except Exception as exc:
+            logger.debug("SAGQ: max query failed: %s", exc)
+            return 0.0
+
+        md = dict(max_rows[0])
+        max_pr = max(float(md["max_pr"]), 1e-8)
+        max_deg = max(float(md["max_deg"]), 1e-8)
+
+        pr_norm = pr_score / max_pr
+        deg_norm = deg_score / max_deg
+
+        # Step 6-7: SA frequency for this fact (Q6)
+        sa_window = f"-{self._config.sa_frequency_window_days} days"
+        try:
+            sa_rows = self._db.execute(
+                "SELECT COUNT(*) as cnt "
+                "FROM activation_cache "
+                "WHERE node_id = ? AND profile_id = ? "
+                "AND created_at > datetime('now', ?)",
+                (fact_id, profile_id, sa_window),
+            )
+            sa_cnt = int(dict(sa_rows[0])["cnt"]) if sa_rows else 0
+        except Exception:
+            sa_cnt = 0
+
+        # Step 7: Max SA frequency (Q5)
+        try:
+            max_sa_rows = self._db.execute(
+                "SELECT COALESCE(MAX(cnt), 1) as max_cnt FROM ("
+                "  SELECT COUNT(*) as cnt "
+                "  FROM activation_cache "
+                "  WHERE profile_id = ? "
+                "  AND created_at > datetime('now', ?) "
+                "  GROUP BY node_id"
+                ")",
+                (profile_id, sa_window),
+            )
+            max_sa = int(dict(max_sa_rows[0])["max_cnt"]) if max_sa_rows else 1
+        except Exception:
+            max_sa = 1
+
+        sa_norm = sa_cnt / max(max_sa, 1)
+
+        # Step 9: Weighted combination
+        cfg = self._config
+        combined = cfg.w_pagerank * pr_norm + cfg.w_degree * deg_norm + cfg.w_sa_freq * sa_norm
+
+        # NaN safety
+        if math.isnan(combined):
+            logger.warning("SAGQ: NaN centrality for %s, defaulting to 0.0", fact_id)
+            return 0.0
+
+        # Step 10: Clamp
+        return max(0.0, min(1.0, combined))

package/src/superlocalmemory/dynamics/eap_scheduler.py

@@ -0,0 +1,276 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under the MIT License - see LICENSE file
+# Part of SuperLocalMemory V3
+
+"""EAP Scheduler -- Embedding-Aware Precision.
+
+Couples Ebbinghaus retention scores to embedding bit-width:
+  R > 0.8  -> 32 (float32, full precision)
+  R > 0.5  ->  8 (int8, sqlite-vec native)
+  R > 0.2  ->  4 (polar 4-bit)
+  R > 0.05 ->  2 (polar 2-bit)
+  R <= 0.05->  0 (forgotten, delete embedding)
+
+The EAP cycle:
+  1. Fetch all facts with retention data + current bit_width
+  2. Map retention to target bit_width
+  3. Execute downgrades (compress) and upgrades (restore)
+  4. Return stats
+
+HR-03: Original float32 NEVER deleted unless keep_float32_backup=False
+       AND fact is in archive/forgotten zone.
+HR-04: Quantization ONLY via EAP scheduler (not ad-hoc).
+
+Part of Qualixar | Author: Varun Pratap Bhardwaj
+License: MIT
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from typing import TYPE_CHECKING
+
+import numpy as np
+from numpy.typing import NDArray
+
+from superlocalmemory.core.config import QuantizationConfig
+
+if TYPE_CHECKING:
+    from superlocalmemory.math.ebbinghaus import EbbinghausCurve
+    from superlocalmemory.storage.database import DatabaseManager
+    from superlocalmemory.storage.quantized_store import QuantizedEmbeddingStore
+
+logger = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# Retention -> bit-width mapping (Section 10, Pattern 4)
+# ---------------------------------------------------------------------------
+
+
+def retention_to_bit_width(retention: float) -> int:
+    """Map Ebbinghaus retention to embedding precision.
+
+    Returns:
+        32, 8, 4, 2, or 0 (deleted).
+    """
+    if retention > 0.8:
+        return 32
+    if retention > 0.5:
+        return 8
+    if retention > 0.2:
+        return 4
+    if retention > 0.05:
+        return 2
+    return 0
+
+
+# ---------------------------------------------------------------------------
+# EAPScheduler
+# ---------------------------------------------------------------------------
+
+
+class EAPScheduler:
+    """Embedding-Aware Precision scheduler.
+
+    Runs periodic cycles that adjust embedding precision based on
+    how well each fact is retained in memory.
+    """
+
+    # No __slots__: allows mock patching of _get_fact_embedding in tests
+
+    def __init__(
+        self,
+        db: DatabaseManager,
+        ebbinghaus: EbbinghausCurve,
+        quantized_store: QuantizedEmbeddingStore,
+        config: QuantizationConfig,
+    ) -> None:
+        """Initialize EAP scheduler.
+
+        No side effects, no DB calls, no file I/O.
+        """
+        self._db = db
+        self._ebbinghaus = ebbinghaus
+        self._quantized_store = quantized_store
+        self._config = config
+
+    def run_eap_cycle(self, profile_id: str) -> dict:
+        """Execute one EAP cycle for a profile.
+
+        Steps:
+          1. Fetch all facts with retention data
+          2. Map retention -> target bit_width
+          3. Execute downgrades/upgrades
+          4. Return stats
+
+        Returns:
+            {total, downgrades, upgrades, skipped, deleted, errors}
+        """
+        stats = {
+            "total": 0,
+            "downgrades": 0,
+            "upgrades": 0,
+            "skipped": 0,
+            "deleted": 0,
+            "errors": 0,
+        }
+
+        # Step 1: Fetch all facts with retention + current bit_width
+        try:
+            rows = self._db.execute(
+                "SELECT r.fact_id, r.retention_score, r.lifecycle_zone, "
+                "  COALESCE(eqm.bit_width, 32) as current_bw "
+                "FROM fact_retention r "
+                "LEFT JOIN embedding_quantization_metadata eqm "
+                "  ON r.fact_id = eqm.fact_id "
+                "WHERE r.profile_id = ?",
+                (profile_id,),
+            )
+        except Exception as exc:
+            logger.error("EAP cycle query failed: %s", exc)
+            stats["errors"] = 1
+            return stats
+
+        if not rows:
+            return stats
+
+        # Step 2+3: Process each fact
+        for row in rows:
+            d = dict(row)
+            fact_id = d["fact_id"]
+            retention = float(d["retention_score"])
+            current_bw = int(d["current_bw"])
+
+            stats["total"] += 1
+
+            target_bw = retention_to_bit_width(retention)
+
+            if target_bw == current_bw:
+                stats["skipped"] += 1
+                continue
+
+            if target_bw == 0:
+                # Forgotten -- mark as deleted
+                self._handle_deletion(fact_id, profile_id)
+                stats["deleted"] += 1
+                continue
+
+            if target_bw < current_bw:
+                # Downgrade -- compress to lower precision
+                success = self._handle_downgrade(
+                    fact_id, profile_id, target_bw,
+                )
+                if success:
+                    stats["downgrades"] += 1
+                else:
+                    stats["errors"] += 1
+            else:
+                # Upgrade -- restore to higher precision (only if float32 exists)
+                success = self._handle_upgrade(
+                    fact_id, profile_id, target_bw,
+                )
+                if success:
+                    stats["upgrades"] += 1
+                else:
+                    stats["skipped"] += 1  # Can't upgrade without float32
+
+        return stats
+
+    # -- Handlers ----------------------------------------------------------
+
+    def _handle_downgrade(
+        self, fact_id: str, profile_id: str, target_bw: int,
+    ) -> bool:
+        """Compress a fact to lower bit_width.
+
+        Fetches original float32 embedding, quantizes, and stores.
+        """
+        embedding = self._get_fact_embedding(fact_id)
+        if embedding is None:
+            logger.info(
+                "No float32 embedding for %s, cannot compress", fact_id,
+            )
+            return False
+
+        return self._quantized_store.compress_fact(
+            fact_id, profile_id, embedding, target_bw,
+        )
+
+    def _handle_upgrade(
+        self, fact_id: str, profile_id: str, target_bw: int,
+    ) -> bool:
+        """Restore a fact to higher precision.
+
+        Only possible if original float32 is still in fact_embeddings.
+        """
+        # For upgrade to float32, just update metadata
+        if target_bw == 32:
+            try:
+                self._db.execute(
+                    "INSERT INTO embedding_quantization_metadata "
+                    "(fact_id, profile_id, quantization_level, bit_width, created_at) "
+                    "VALUES (?, ?, 'float32', 32, datetime('now')) "
+                    "ON CONFLICT(fact_id) DO UPDATE SET "
+                    "  quantization_level = 'float32', "
+                    "  bit_width = 32",
+                    (fact_id, profile_id),
+                )
+                return True
+            except Exception as exc:
+                logger.error("Upgrade to float32 failed for %s: %s", fact_id, exc)
+                return False
+
+        # For upgrade to int8 from polar, re-compress at higher precision
+        embedding = self._get_fact_embedding(fact_id)
+        if embedding is None:
+            return False
+
+        return self._quantized_store.compress_fact(
+            fact_id, profile_id, embedding, target_bw,
+        )
+
+    def _handle_deletion(self, fact_id: str, profile_id: str) -> None:
+        """Mark an embedding as deleted (forgotten).
+
+        HR-03: Only deletes if keep_float32_backup is False.
+        """
+        try:
+            self._db.execute(
+                "INSERT INTO embedding_quantization_metadata "
+                "(fact_id, profile_id, quantization_level, bit_width, created_at) "
+                "VALUES (?, ?, 'deleted', 0, datetime('now')) "
+                "ON CONFLICT(fact_id) DO UPDATE SET "
+                "  quantization_level = 'deleted', "
+                "  bit_width = 0",
+                (fact_id, profile_id),
+            )
+        except Exception as exc:
+            logger.error("Delete metadata failed for %s: %s", fact_id, exc)
+
+    def _get_fact_embedding(self, fact_id: str) -> NDArray | None:
+        """Retrieve original float32 embedding for a fact.
+
+        Tries embedding_metadata -> fact_embeddings (vec0 table).
+        Falls back to atomic_facts.embedding JSON column.
+        """
+        # Try atomic_facts.embedding (JSON column)
+        try:
+            rows = self._db.execute(
+                "SELECT embedding FROM atomic_facts WHERE fact_id = ?",
+                (fact_id,),
+            )
+            if rows:
+                raw = dict(rows[0]).get("embedding")
+                if raw and raw != "null":
+                    data = json.loads(raw) if isinstance(raw, str) else raw
+                    if data:
+                        return np.array(data, dtype=np.float64)
+        except Exception as exc:
+            logger.debug(
+                "Could not load embedding from atomic_facts for %s: %s",
+                fact_id, exc,
+            )
+
+        return None