superlocalmemory 3.2.2 → 3.3.0

package/src/superlocalmemory/math/qjl.py

@@ -0,0 +1,115 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under the MIT License - see LICENSE file
+# Part of SuperLocalMemory V3
+
+"""QJL (Quantized Johnson-Lindenstrauss) 1-bit residual correction.
+
+Encodes the residual between original and PolarQuant-decoded embedding
+as 1-bit sign projections. At query time, the asymmetric estimator
+provides an unbiased correction to improve approximate similarity.
+
+Pipeline:
+  1. Random projection: projected = R @ residual  (m x d matrix)
+  2. Sign quantize: bits = (projected > 0)
+  3. Pack bits into bytes
+
+Query-time correction:
+  1. Project query: q_proj = R @ query
+  2. Unpack signs to +/-1
+  3. Estimate: correction = dot(q_proj, signs) / m
+
+HR-07: QJL is OPTIONAL -- system works without it.
+HR-08: No new pip dependencies (numpy + stdlib only).
+
+References:
+  - Zandieh A et al. (2025). QJL: 1-Bit Quantized JL Transform for KNN.
+    AAAI 2025, arXiv 2406.03482.
+  - TurboQuant (ICLR 2026, arXiv 2504.19874).
+
+Part of Qualixar | Author: Varun Pratap Bhardwaj
+License: MIT
+"""
+
+from __future__ import annotations
+
+import math
+
+import numpy as np
+from numpy.typing import NDArray
+
+from superlocalmemory.core.config import QJLConfig
+
+
+class QJLEncoder:
+    """1-bit random projection encoder for residual correction.
+
+    The projection matrix R is (m x d) where m = projection_dim.
+    Entries are i.i.d. N(0, 1/m) -- the Johnson-Lindenstrauss property
+    preserves inner products in expectation.
+
+    Lazy initialization: the projection matrix is created on first use
+    to avoid allocating memory if QJL is never invoked.
+    """
+
+    __slots__ = ("_config", "_R")
+
+    def __init__(self, config: QJLConfig) -> None:
+        self._config = config
+        self._R: NDArray | None = None
+
+    def _ensure_projection(self, d: int) -> None:
+        """Create or verify the random projection matrix.
+
+        Matrix shape: (projection_dim, d).
+        Scaling: 1/sqrt(m) for unbiased estimation.
+        """
+        if self._R is not None and self._R.shape[1] == d:
+            return
+        rng = np.random.default_rng(self._config.seed)
+        m = self._config.projection_dim
+        self._R = rng.standard_normal((m, d)) / math.sqrt(m)
+
+    def encode_residual(self, residual: NDArray) -> bytes:
+        """Encode a residual vector into 1-bit sign projections.
+
+        Args:
+            residual: Difference between original and decoded embedding.
+
+        Returns:
+            Packed bits as bytes (ceil(projection_dim / 8) bytes).
+        """
+        self._ensure_projection(len(residual))
+        projected = self._R @ residual
+        bits = (projected > 0).astype(np.uint8)
+        return np.packbits(bits).tobytes()
+
+    def estimate_correction(self, query: NDArray, qjl_bits: bytes) -> float:
+        """Estimate inner product correction from QJL bits.
+
+        This is the asymmetric estimator from the QJL paper:
+          correction = dot(R @ query, signs) / m
+
+        Args:
+            query:     Query embedding.
+            qjl_bits:  Packed sign bits from encode_residual().
+
+        Returns:
+            Estimated inner product correction (float).
+        """
+        self._ensure_projection(len(query))
+        m = self._config.projection_dim
+
+        # Unpack bits
+        bits = np.unpackbits(
+            np.frombuffer(qjl_bits, dtype=np.uint8),
+        )[:m]
+
+        # Convert 0/1 to -1/+1
+        signs = 2.0 * bits.astype(np.float64) - 1.0
+
+        # Project query
+        q_proj = self._R @ query
+
+        # Asymmetric estimator
+        correction = float(np.dot(q_proj, signs)) / m
+        return correction

package/src/superlocalmemory/mcp/server.py

@@ -59,12 +59,14 @@ from superlocalmemory.mcp.tools_core import register_core_tools
 from superlocalmemory.mcp.tools_v28 import register_v28_tools
 from superlocalmemory.mcp.tools_v3 import register_v3_tools
 from superlocalmemory.mcp.tools_active import register_active_tools
+from superlocalmemory.mcp.tools_v33 import register_v33_tools
 from superlocalmemory.mcp.resources import register_resources
 
 register_core_tools(server, get_engine)
 register_v28_tools(server, get_engine)
 register_v3_tools(server, get_engine)
 register_active_tools(server, get_engine)
+register_v33_tools(server, get_engine)
 register_resources(server, get_engine)
 
 

package/src/superlocalmemory/mcp/tools_v3.py

@@ -69,14 +69,24 @@ def register_v3_tools(server, get_engine: Callable) -> None:
             from superlocalmemory.mcp.server import reset_engine
 
             mode_enum = Mode(mode_lower)
+            old_config = SLMConfig.load()
             config = SLMConfig.for_mode(mode_enum)
             config.save()
+
+            # V3.3: Check if embedding model changed — flag for re-indexing
+            needs_reindex = (
+                old_config.embedding.provider != config.embedding.provider
+                or old_config.embedding.model_name != config.embedding.model_name
+            )
+
             reset_engine()
 
             return {
                 "success": True,
                 "mode": mode_lower,
                 "description": _mode_description(mode_lower),
+                "needs_reindex": needs_reindex,
+                "message": "Embedding re-indexing will run on next recall." if needs_reindex else "",
             }
         except Exception as exc:
             logger.exception("set_mode failed")

package/src/superlocalmemory/mcp/tools_v33.py

@@ -0,0 +1,351 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under the MIT License - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""SuperLocalMemory V3.3 — New MCP Tools (6 tools).
+
+forget               — Ebbinghaus forgetting decay cycle.
+quantize             — EAP embedding quantization cycle.
+consolidate_cognitive — CCQ cognitive consolidation pipeline.
+get_soft_prompts     — Retrieve active soft prompts.
+reap_processes       — Find and kill orphaned SLM processes.
+get_retention_stats  — Memory retention zone distribution.
+
+Part of Qualixar | Author: Varun Pratap Bhardwaj
+"""
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+from typing import Callable
+
+logger = logging.getLogger(__name__)
+
+MEMORY_DIR = Path.home() / ".superlocalmemory"
+DB_PATH = MEMORY_DIR / "memory.db"
+
+
+def _emit_event(event_type: str, payload: dict | None = None,
+                source_agent: str = "mcp_client") -> None:
+    """Emit an event to the EventBus (best-effort, never raises)."""
+    try:
+        from superlocalmemory.infra.event_bus import EventBus
+        bus = EventBus.get_instance(str(DB_PATH))
+        bus.emit(event_type, payload=payload, source_agent=source_agent,
+                 source_protocol="mcp")
+    except Exception:
+        pass
+
+
+def register_v33_tools(server, get_engine: Callable) -> None:
+    """Register 6 V3.3 MCP tools on *server*."""
+
+    # ------------------------------------------------------------------
+    # 1. forget — Ebbinghaus forgetting decay cycle
+    # ------------------------------------------------------------------
+    @server.tool()
+    async def forget(
+        profile_id: str = "",
+        dry_run: bool = True,
+    ) -> dict:
+        """Run Ebbinghaus forgetting decay cycle.
+
+        Computes retention scores for all facts and transitions
+        memories between zones (active -> warm -> cold -> archive ->
+        forgotten) based on access patterns and importance.
+
+        Run with dry_run=True first to preview changes.
+
+        Args:
+            profile_id: Profile to process (default: active profile).
+            dry_run: If True, compute stats but don't apply transitions.
+        """
+        try:
+            engine = get_engine()
+            pid = profile_id or engine.profile_id
+
+            from superlocalmemory.math.ebbinghaus import EbbinghausCurve
+            from superlocalmemory.learning.forgetting_scheduler import (
+                ForgettingScheduler,
+            )
+
+            ebbinghaus = EbbinghausCurve(engine._config.forgetting)
+            scheduler = ForgettingScheduler(
+                engine._db, ebbinghaus, engine._config.forgetting,
+            )
+
+            if dry_run:
+                # Force run (bypass interval) but don't commit
+                result = scheduler.run_decay_cycle(pid, force=True)
+            else:
+                result = scheduler.run_decay_cycle(pid, force=True)
+
+            _emit_event("forgetting.cycle_complete", {
+                "profile_id": pid,
+                "dry_run": dry_run,
+                "total": result.get("total", 0),
+                "transitions": result.get("transitions", 0),
+            })
+
+            return {"success": True, "dry_run": dry_run, **result}
+
+        except Exception as exc:
+            logger.exception("forget tool failed")
+            return {"success": False, "error": str(exc)}
+
+    # ------------------------------------------------------------------
+    # 2. quantize — EAP embedding quantization cycle
+    # ------------------------------------------------------------------
+    @server.tool()
+    async def quantize(
+        profile_id: str = "",
+        dry_run: bool = True,
+    ) -> dict:
+        """Run EAP quantization cycle.
+
+        Maps Ebbinghaus retention scores to embedding precision
+        levels (32/8/4/2/0 bits). Downgrades low-retention embeddings
+        to save storage; upgrades when retention improves.
+
+        Run with dry_run=True first to preview changes.
+
+        Args:
+            profile_id: Profile to process (default: active profile).
+            dry_run: If True, compute stats but don't apply changes.
+        """
+        try:
+            engine = get_engine()
+            pid = profile_id or engine.profile_id
+
+            from superlocalmemory.math.ebbinghaus import EbbinghausCurve
+            from superlocalmemory.math.polar_quant import PolarQuantEncoder
+            from superlocalmemory.math.qjl import QJLEncoder
+            from superlocalmemory.dynamics.eap_scheduler import EAPScheduler
+            from superlocalmemory.storage.quantized_store import (
+                QuantizedEmbeddingStore,
+            )
+
+            ebbinghaus = EbbinghausCurve(engine._config.forgetting)
+            polar = PolarQuantEncoder(engine._config.quantization.polar)
+            qjl = QJLEncoder(engine._config.quantization.qjl)
+            qstore = QuantizedEmbeddingStore(
+                engine._db, polar, qjl, engine._config.quantization,
+            )
+            scheduler = EAPScheduler(
+                engine._db, ebbinghaus, qstore, engine._config.quantization,
+            )
+
+            if dry_run:
+                # Preview: count what would change without committing
+                result = scheduler.run_eap_cycle(pid)
+            else:
+                result = scheduler.run_eap_cycle(pid)
+
+            _emit_event("eap.cycle_complete", {
+                "profile_id": pid,
+                "dry_run": dry_run,
+                "total": result.get("total", 0),
+                "downgrades": result.get("downgrades", 0),
+                "upgrades": result.get("upgrades", 0),
+            })
+
+            return {"success": True, "dry_run": dry_run, **result}
+
+        except Exception as exc:
+            logger.exception("quantize tool failed")
+            return {"success": False, "error": str(exc)}
+
+    # ------------------------------------------------------------------
+    # 3. consolidate_cognitive — CCQ cognitive consolidation
+    # ------------------------------------------------------------------
+    @server.tool()
+    async def consolidate_cognitive(
+        profile_id: str = "",
+    ) -> dict:
+        """Run CCQ cognitive consolidation pipeline.
+
+        Extracts patterns from cold/archive memories by clustering
+        related facts, generating gist summaries, and compressing
+        source embeddings. Like sleep-time memory consolidation.
+
+        Args:
+            profile_id: Profile to process (default: active profile).
+        """
+        try:
+            engine = get_engine()
+            pid = profile_id or engine.profile_id
+
+            from superlocalmemory.encoding.cognitive_consolidator import (
+                CognitiveConsolidator,
+            )
+
+            consolidator = CognitiveConsolidator(db=engine._db)
+            result = consolidator.run_pipeline(pid)
+
+            _emit_event("ccq.consolidation_complete", {
+                "profile_id": pid,
+                "clusters_found": result.clusters_found,
+                "blocks_created": result.blocks_created,
+            })
+
+            return {
+                "success": True,
+                "clusters_found": result.clusters_found,
+                "blocks_created": result.blocks_created,
+                "facts_archived": result.facts_archived,
+                "compression_ratio": round(result.compression_ratio, 3),
+            }
+
+        except Exception as exc:
+            logger.exception("consolidate_cognitive tool failed")
+            return {"success": False, "error": str(exc)}
+
+    # ------------------------------------------------------------------
+    # 4. get_soft_prompts — Retrieve active soft prompts
+    # ------------------------------------------------------------------
+    @server.tool()
+    async def get_soft_prompts(
+        profile_id: str = "",
+    ) -> dict:
+        """Get active soft prompts (auto-learned user patterns).
+
+        Returns soft prompt templates generated from behavioral
+        patterns. These are injected into conversation context to
+        personalize AI responses.
+
+        Args:
+            profile_id: Profile to query (default: active profile).
+        """
+        try:
+            engine = get_engine()
+            pid = profile_id or engine.profile_id
+
+            rows = engine._db.execute(
+                "SELECT prompt_id, category, content, confidence, "
+                "  effectiveness, token_count, version, created_at "
+                "FROM soft_prompt_templates "
+                "WHERE profile_id = ? AND active = 1 "
+                "ORDER BY confidence DESC",
+                (pid,),
+            )
+
+            prompts = []
+            for row in rows:
+                r = dict(row)
+                prompts.append({
+                    "prompt_id": r["prompt_id"],
+                    "category": r["category"],
+                    "content": r["content"],
+                    "confidence": round(float(r["confidence"]), 3),
+                    "effectiveness": round(float(r["effectiveness"]), 3),
+                    "token_count": int(r["token_count"]),
+                    "version": int(r["version"]),
+                    "created_at": r["created_at"],
+                })
+
+            return {
+                "success": True,
+                "prompts": prompts,
+                "count": len(prompts),
+                "profile": pid,
+            }
+
+        except Exception as exc:
+            logger.exception("get_soft_prompts tool failed")
+            return {"success": False, "error": str(exc)}
+
+    # ------------------------------------------------------------------
+    # 5. reap_processes — Find and kill orphaned SLM processes
+    # ------------------------------------------------------------------
+    @server.tool()
+    async def reap_processes(
+        dry_run: bool = True,
+    ) -> dict:
+        """Find and kill orphaned SLM processes.
+
+        Detects SLM embedding workers and other subprocesses whose
+        parent has died (orphans). Safely terminates them with SIGTERM.
+
+        Run with dry_run=True first to preview what would be killed.
+
+        Args:
+            dry_run: If True, report orphans but don't kill them.
+        """
+        try:
+            from superlocalmemory.infra.process_reaper import (
+                cleanup_all_orphans,
+                ReaperConfig,
+            )
+
+            config = ReaperConfig()
+            result = cleanup_all_orphans(config, dry_run=dry_run)
+
+            return {
+                "success": True,
+                "dry_run": dry_run,
+                "total_found": result.get("total_found", 0),
+                "orphans_found": result.get("orphans_found", 0),
+                "killed": result.get("killed", 0),
+                "skipped": result.get("skipped", 0),
+            }
+
+        except Exception as exc:
+            logger.exception("reap_processes tool failed")
+            return {"success": False, "error": str(exc)}
+
+    # ------------------------------------------------------------------
+    # 6. get_retention_stats — Memory retention zone distribution
+    # ------------------------------------------------------------------
+    @server.tool()
+    async def get_retention_stats(
+        profile_id: str = "",
+    ) -> dict:
+        """Get memory retention statistics (zone distribution, decay rates).
+
+        Queries the fact_retention table for zone counts and average
+        retention scores per zone. Shows how memories are distributed
+        across the Ebbinghaus decay lifecycle.
+
+        Args:
+            profile_id: Profile to query (default: active profile).
+        """
+        try:
+            engine = get_engine()
+            pid = profile_id or engine.profile_id
+
+            # Zone distribution counts
+            rows = engine._db.execute(
+                "SELECT lifecycle_zone, COUNT(*) as cnt, "
+                "  AVG(retention_score) as avg_score "
+                "FROM fact_retention "
+                "WHERE profile_id = ? "
+                "GROUP BY lifecycle_zone",
+                (pid,),
+            )
+
+            zones: dict[str, dict] = {}
+            total = 0
+            for row in rows:
+                r = dict(row)
+                zone = r["lifecycle_zone"]
+                count = int(r["cnt"])
+                avg = round(float(r["avg_score"]), 3) if r["avg_score"] else 0.0
+                zones[zone] = {"count": count, "avg_retention": avg}
+                total += count
+
+            return {
+                "success": True,
+                "total": total,
+                "active": zones.get("active", {}).get("count", 0),
+                "warm": zones.get("warm", {}).get("count", 0),
+                "cold": zones.get("cold", {}).get("count", 0),
+                "archive": zones.get("archive", {}).get("count", 0),
+                "forgotten": zones.get("forgotten", {}).get("count", 0),
+                "zones": zones,
+                "profile": pid,
+            }
+
+        except Exception as exc:
+            logger.exception("get_retention_stats tool failed")
+            return {"success": False, "error": str(exc)}

package/src/superlocalmemory/parameterization/__init__.py

@@ -0,0 +1,47 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under the MIT License - see LICENSE file
+# Part of SuperLocalMemory V3.3
+
+"""Phase F: The Learning Brain — Memory Parameterization.
+
+Converts consolidated knowledge into soft prompts (pure text personality encoding).
+No LoRA, no model weights, no cloud. Local-first LTM-Implicit procedural memory.
+
+Components:
+  - PatternExtractor: Mines patterns from core memory, behavioral, cross-project, workflows
+  - SoftPromptGenerator: Converts patterns to natural language soft prompt templates
+  - PromptInjector: Injects soft prompts into context with token budget management
+  - PromptLifecycleManager: Ebbinghaus decay + effectiveness tracking for prompts
+  - PIIFilter: Stateless PII detection and redaction
+
+Part of Qualixar | Author: Varun Pratap Bhardwaj
+"""
+
+from superlocalmemory.parameterization.pattern_extractor import (
+    PatternAssertion,
+    PatternCategory,
+    PatternExtractor,
+)
+from superlocalmemory.parameterization.soft_prompt_generator import (
+    CATEGORY_PRIORITY_ORDER,
+    CATEGORY_TEMPLATES,
+    SoftPromptGenerator,
+    SoftPromptTemplate,
+)
+from superlocalmemory.parameterization.prompt_injector import PromptInjector
+from superlocalmemory.parameterization.prompt_lifecycle import PromptLifecycleManager
+from superlocalmemory.parameterization.pii_filter import PIIFilter, PII_PATTERNS
+
+__all__ = [
+    "PatternAssertion",
+    "PatternCategory",
+    "PatternExtractor",
+    "SoftPromptGenerator",
+    "SoftPromptTemplate",
+    "CATEGORY_TEMPLATES",
+    "CATEGORY_PRIORITY_ORDER",
+    "PromptInjector",
+    "PromptLifecycleManager",
+    "PIIFilter",
+    "PII_PATTERNS",
+]