superlocalmemory 3.7.7 → 3.8.0

package/src/superlocalmemory/core/engine.py

@@ -276,6 +276,18 @@ class MemoryEngine:
                 )
                 self._llm = None
 
+        # H-03 (3.7.9): surface mode-capability degradation explicitly at
+        # startup. validate_mode_config existed but was never called, so Mode B
+        # silently using rule-based extraction after Ollama vanished (update,
+        # restart, port conflict) went unwarned. Each mode only checks its own
+        # capabilities, so passing the single llm-availability signal is safe.
+        from superlocalmemory.core.modes import validate_mode_config
+        _llm_up = getattr(self, "_llm", None) is not None
+        for _warning in validate_mode_config(
+            self._config.mode, has_ollama=_llm_up, has_cloud_llm=_llm_up,
+        ):
+            logger.warning("Mode config: %s", _warning)
+
         from superlocalmemory.trust.scorer import TrustScorer
         from superlocalmemory.trust.provenance import ProvenanceTracker
         from superlocalmemory.compliance.eu_ai_act import EUAIActChecker
@@ -374,7 +386,9 @@ class MemoryEngine:
             return
 
         base_dir = self._config.base_dir
-        pending = get_pending(base_dir, limit=20)
+        # Only drain items enqueued under THIS engine's profile — a queued
+        # memory must never materialize under a profile it was not written for.
+        pending = get_pending(base_dir, limit=20, profile_id=self.profile_id)
         if not pending:
             return
 
@@ -599,6 +613,7 @@ class MemoryEngine:
         *,
         include_global: bool | None = None,
         include_shared: bool | None = None,
+        window: str | tuple[str, str] | None = None,
     ) -> RecallResponse:
         """Recall relevant facts for a query.
 
@@ -650,6 +665,7 @@ class MemoryEngine:
                 fast=fast,
                 include_global=include_global,
                 include_shared=include_shared,
+                window=window,
             )
         except Exception as exc:
             from superlocalmemory.infra.local_diagnostics import record_operation

package/src/superlocalmemory/core/engine_ingestion.py

@@ -11,9 +11,13 @@ implementation of queryable projection and complete derivation.
 from __future__ import annotations
 
 import hashlib
+import logging
+import os
 import uuid
 from typing import TYPE_CHECKING
 
+logger = logging.getLogger(__name__)
+
 from superlocalmemory.core.ingestion_command import (
     IngestionCommand,
     IngestionOperation,
@@ -30,6 +34,21 @@ if TYPE_CHECKING:
 _PREBUILT_FACT_KEY = "_slm_prebuilt_fact_v1"
 
 
+def _pii_redaction_enabled(engine: "MemoryEngine") -> bool:
+    """C4: opt-in PII redaction on ingest.
+
+    On when the engine config sets ``pii_redaction`` truthy OR the
+    ``SLM_PII_REDACTION`` env var is set (1/on/true/yes). Default OFF — personal
+    use is unchanged; team/company operators opt in.
+    """
+    cfg = getattr(engine, "_config", None)
+    if cfg is not None and getattr(cfg, "pii_redaction", False):
+        return True
+    return os.environ.get("SLM_PII_REDACTION", "").strip().lower() in (
+        "1", "on", "true", "yes",
+    )
+
+
 def content_passes_admission(content: str) -> bool:
     """Return whether raw content is eligible to reach durable ingestion.
 
@@ -140,6 +159,16 @@ def canonical_store(
             error=ValueError("content rejected by local admission policy"),
         )
         return []
+    # C4: opt-in PII redaction. When enabled (config.pii_redaction or
+    # SLM_PII_REDACTION), scrub personal identifiers BEFORE the content is
+    # extracted, embedded, or persisted — nothing sensitive ever reaches disk.
+    if _pii_redaction_enabled(engine):
+        from superlocalmemory.core.pii import redact_pii
+
+        scrubbed, n_pii = redact_pii(content)
+        if n_pii:
+            content = scrubbed
+            logger.info("PII redaction: scrubbed %d identifier(s) on ingest", n_pii)
     try:
         command = build_engine_ingestion_command(engine)
         receipt = command.submit(IngestionRequest(

package/src/superlocalmemory/core/engine_wiring.py

@@ -588,6 +588,19 @@ def init_retrieval(
     except Exception as exc:
         logger.debug("Forgetting filter registration failed: %s", exc)
 
+    # Phase 4 (T1): Register bi-temporal validity filter. Drops superseded /
+    # system-invalidated facts from retrieval so contradicted memories never
+    # resurface. Pure SQL, safe in all modes; no-op until a fact is invalidated.
+    try:
+        from superlocalmemory.retrieval.temporal_validity_filter import (
+            register_temporal_validity_filter,
+        )
+        register_temporal_validity_filter(
+            engine._registry, db, config.temporal_validator,
+        )
+    except Exception as exc:
+        logger.debug("Temporal validity filter registration failed: %s", exc)
+
     return engine
 
 

package/src/superlocalmemory/core/entity_community.py

@@ -0,0 +1,178 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under AGPL-3.0-or-later - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""Entity-community backbone (Wave Q) — the single principled clustering spine.
+
+Best-in-market memory graphs (GraphRAG, Graphiti/Zep) cluster the ENTITY
+graph, not the raw fact/chunk graph. SLM's fact graph is capped-sparse
+(5 edges/entity), so a fact-level clustering fragments. This module builds an
+entity co-occurrence graph (two entities are linked when they appear together
+in a fact, weighted by co-occurrence count) and runs Louvain community
+detection over it — the correct target and, since entities are far fewer than
+facts, a cheaper computation.
+
+The resulting entity communities are the shared backbone for:
+  - Q2 community summaries (one synthesized report per community), and
+  - Q3 progressive abstraction (scenario/persona tiers + drill-down).
+
+Runs in the background (consolidation lane), never on the hot recall path.
+Fail-open and idempotent: a recompute fully replaces a profile's rows.
+
+Part of Qualixar | Author: Varun Pratap Bhardwaj
+License: AGPL-3.0-or-later
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from collections import Counter, defaultdict
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+
+class EntityCommunityBuilder:
+    """Build + persist entity communities via Louvain over co-occurrence."""
+
+    def __init__(
+        self,
+        db: Any,
+        min_community_size: int = 2,
+        resolution: float = 1.0,
+        seed: int = 42,
+    ) -> None:
+        self._db = db
+        self._min_size = max(2, int(min_community_size))
+        self._resolution = float(resolution)
+        self._seed = int(seed)
+
+    # ------------------------------------------------------------------
+    # Detection
+    # ------------------------------------------------------------------
+
+    def _cooccurrence(self, profile_id: str) -> Counter:
+        """Count entity pairs that co-occur within a fact (per profile)."""
+        rows = self._db.execute(
+            "SELECT canonical_entities_json FROM atomic_facts "
+            "WHERE profile_id = ?",
+            (profile_id,),
+        )
+        weights: Counter = Counter()
+        for row in rows:
+            raw = dict(row).get("canonical_entities_json")
+            if not raw:
+                continue
+            try:
+                parsed = json.loads(raw)
+            except (ValueError, TypeError):
+                continue
+            if not isinstance(parsed, list):
+                continue
+            ents = sorted({str(e).strip() for e in parsed if str(e).strip()})
+            for i in range(len(ents)):
+                for j in range(i + 1, len(ents)):
+                    weights[(ents[i], ents[j])] += 1
+        return weights
+
+    def detect(self, profile_id: str) -> dict[str, int]:
+        """Return {entity_id -> community_id}; communities below min size drop."""
+        weights = self._cooccurrence(profile_id)
+        if not weights:
+            return {}
+
+        import networkx as nx
+
+        g = nx.Graph()
+        for (a, b), w in weights.items():
+            g.add_edge(a, b, weight=w)
+
+        try:
+            from networkx.algorithms.community import louvain_communities
+
+            communities = louvain_communities(
+                g, weight="weight", resolution=self._resolution, seed=self._seed,
+            )
+        except Exception as exc:  # pragma: no cover - fallback path
+            logger.debug(
+                "Louvain unavailable/failed (%s); using connected components",
+                exc,
+            )
+            communities = nx.connected_components(g)
+
+        result: dict[str, int] = {}
+        cid = 0
+        for comm in communities:
+            members = list(comm)
+            if len(members) < self._min_size:
+                continue
+            for node in members:
+                result[node] = cid
+            cid += 1
+        return result
+
+    # ------------------------------------------------------------------
+    # Persistence
+    # ------------------------------------------------------------------
+
+    def compute_and_store(self, profile_id: str) -> dict[str, int]:
+        """Detect communities and replace this profile's stored rows."""
+        mapping = self.detect(profile_id)
+        try:
+            self._db.execute(
+                "DELETE FROM entity_communities WHERE profile_id = ?",
+                (profile_id,),
+            )
+            for entity_id, community_id in mapping.items():
+                self._db.execute(
+                    "INSERT OR REPLACE INTO entity_communities "
+                    "(profile_id, entity_id, community_id, computed_at) "
+                    "VALUES (?, ?, ?, datetime('now'))",
+                    (profile_id, entity_id, community_id),
+                )
+        except Exception as exc:
+            logger.debug("entity_communities persist failed: %s", exc)
+
+        return {
+            "entity_count": len(mapping),
+            "community_count": len(set(mapping.values())),
+        }
+
+    # ------------------------------------------------------------------
+    # Read API
+    # ------------------------------------------------------------------
+
+    def get_communities(self, profile_id: str) -> dict[int, list[str]]:
+        """Return {community_id -> [entity_id, ...]} for a profile."""
+        try:
+            rows = self._db.execute(
+                "SELECT entity_id, community_id FROM entity_communities "
+                "WHERE profile_id = ? ORDER BY community_id",
+                (profile_id,),
+            )
+        except Exception as exc:
+            logger.debug("get_communities failed: %s", exc)
+            return {}
+        out: dict[int, list[str]] = defaultdict(list)
+        for row in rows:
+            d = dict(row)
+            out[int(d["community_id"])].append(str(d["entity_id"]))
+        return dict(out)
+
+    def get_community_for_entity(
+        self, entity_id: str, profile_id: str,
+    ) -> int | None:
+        """Return the community id for one entity, or None."""
+        try:
+            rows = self._db.execute(
+                "SELECT community_id FROM entity_communities "
+                "WHERE profile_id = ? AND entity_id = ?",
+                (profile_id, entity_id),
+            )
+        except Exception as exc:
+            logger.debug("get_community_for_entity failed: %s", exc)
+            return None
+        for row in rows:
+            return int(dict(row)["community_id"])
+        return None

package/src/superlocalmemory/core/graph_analyzer.py

@@ -211,6 +211,43 @@ class GraphAnalyzer:
                 result[node] = comm_id
         return result
 
+    def detect_communities_louvain(
+        self,
+        graph: Any = None,
+        profile_id: str = "",
+    ) -> dict[str, int]:
+        """Detect communities via Louvain (modularity-optimizing).
+
+        Higher quality than Label Propagation (deterministic with a seed,
+        no giant-community collapse), pure-Python via networkx — no extra
+        binary deps. Falls back to Label Propagation if unavailable.
+        """
+        import networkx as nx
+
+        if graph is None:
+            graph = self._build_networkx_graph(profile_id)
+        if graph.number_of_nodes() == 0:
+            return {}
+
+        undirected = graph.to_undirected()
+        try:
+            from networkx.algorithms.community import louvain_communities
+
+            communities = louvain_communities(
+                undirected, weight="weight", seed=42,
+            )
+        except Exception as exc:
+            logger.debug(
+                "Louvain unavailable/failed (%s); using Label Propagation", exc,
+            )
+            return self.detect_communities(graph, profile_id)
+
+        result: dict[str, int] = {}
+        for comm_id, community in enumerate(communities):
+            for node in community:
+                result[node] = comm_id
+        return result
+
     # ── v3.4.1: Leiden Community Detection ────────────────────────
 
     def detect_communities_leiden(
@@ -234,9 +271,9 @@ class GraphAnalyzer:
             import igraph
         except ImportError:
             logger.info(
-                "leidenalg not installed, using Label Propagation fallback",
+                "leidenalg not installed, using Louvain fallback",
             )
-            return self.detect_communities(graph, profile_id)
+            return self.detect_communities_louvain(graph, profile_id)
 
         # Convert DiGraph -> undirected -> igraph
         undirected = graph.to_undirected()

package/src/superlocalmemory/core/graph_pruner.py

@@ -350,8 +350,12 @@ def _cap_node_degree(
     Algorithm (single-pass window function — no Python loops):
       1. ROW_NUMBER() OVER (PARTITION BY source_id ORDER BY weight DESC) ranks
          every edge per node in one full table scan.
-      2. Edges with rn > max_degree are deleted in a single DELETE statement.
-      Requires SQLite 3.25+ (window functions). System is on 3.53.1.
+      2. Excess edge IDs are collected in a reusable temp table.
+      3. A single DELETE statement removes them; rowcount is returned.
+      4. The temp table is created once and cleared via DELETE (not DROP) —
+         DROP TABLE acquires an EXCLUSIVE lock that conflicts with concurrent
+         writers, causing "database is locked".  Using CREATE...IF NOT EXISTS
+         plus DELETE FROM avoids that conflict while preserving rowcount.
     """
     # gi-03: cap BOTH out-degree (PARTITION BY source_id) AND in-degree
     # (PARTITION BY target_id). Previously only out-degree was capped, so hub
@@ -379,9 +383,13 @@ def _cap_node_degree(
         )
         return excess
 
-    # Step 1: collect edges exceeding the cap in either direction (one pass).
-    c.execute("DROP TABLE IF EXISTS _slm_cap_del")
-    c.execute("CREATE TEMP TABLE _slm_cap_del (edge_id TEXT PRIMARY KEY)")
+    # CREATE IF NOT EXISTS + DELETE FROM instead of DROP + CREATE.
+    # DROP TABLE acquires EXCLUSIVE which conflicts with concurrent writers.
+    # CREATE IF NOT EXISTS is idempotent; DELETE FROM clears prior contents.
+    c.execute(
+        "CREATE TEMP TABLE IF NOT EXISTS _slm_cap_del (edge_id TEXT PRIMARY KEY)"
+    )
+    c.execute("DELETE FROM _slm_cap_del")
     c.execute(
         """
         INSERT OR IGNORE INTO _slm_cap_del (edge_id)
@@ -396,7 +404,6 @@ def _cap_node_degree(
         (profile_id, max_degree, max_degree),
     )
 
-    # Step 2: delete the over-cap edges (single DELETE).
     c.execute(
         """
         DELETE FROM graph_edges
@@ -407,8 +414,6 @@ def _cap_node_degree(
     )
     deleted = c.rowcount
 
-    c.execute("DROP TABLE IF EXISTS _slm_cap_del")
-
     logger.info(
         "_cap_node_degree: deleted %d low-weight edges (max_degree=%d, in+out capped)",
         deleted, max_degree,

package/src/superlocalmemory/core/key_expander.py

@@ -0,0 +1,138 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under AGPL-3.0-or-later - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""Fact-augmented key expansion (Phase 4, T3b).
+
+Generates *alternate keys* for a fact — synonyms, aliases, and paraphrases —
+that get indexed in ``fact_expansion_fts`` and UNION'd into BM25 retrieval, so a
+query for "automobile" or "the Big Apple" can match a fact that only says "car"
+or "NYC".
+
+Two tiers, matching the rest of SLM:
+  * Mode A (zero-LLM): pulls the fact's resolved entities' canonical names and
+    aliases from SLM's own entity graph — no model, no cost, and it reuses the
+    entity resolution already done at ingest.
+  * Mode B/C: additionally asks the LLM for a few paraphrase keywords (own
+    prompt, fail-open, bounded).
+
+Keys already present in the fact's content are dropped — indexing them twice
+adds nothing. Returns a single space-joined string ready for the FTS row.
+
+Part of Qualixar | Author: Varun Pratap Bhardwaj
+License: AGPL-3.0-or-later
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from superlocalmemory.storage.database import DatabaseManager
+    from superlocalmemory.storage.models import AtomicFact
+
+logger = logging.getLogger(__name__)
+
+# Bound the LLM enrichment so a pathological response can't bloat the index.
+_MAX_LLM_KEYS = 8
+_MAX_KEY_LEN = 60
+
+
+class KeyExpander:
+    """Produces alternate search keys for a fact (T3b)."""
+
+    __slots__ = ("_db", "_llm")
+
+    def __init__(self, db: DatabaseManager, llm: Any = None) -> None:
+        self._db = db
+        self._llm = llm
+
+    def expand(self, fact: AtomicFact, profile_id: str, mode: str = "a") -> str:
+        """Return space-joined alternate keys for ``fact`` (may be empty)."""
+        keys: set[str] = set()
+        keys |= self._alias_keys(fact, profile_id)
+        if mode in ("b", "c") and self._llm_available():
+            keys |= self._llm_keys(fact)
+
+        content_low = (getattr(fact, "content", "") or "").lower()
+        cleaned = {
+            k.strip() for k in keys
+            if k and k.strip() and len(k.strip()) <= _MAX_KEY_LEN
+            and k.strip().lower() not in content_low
+        }
+        return " ".join(sorted(cleaned))
+
+    # -- Mode A: entity aliases from SLM's own entity graph ------------------
+
+    def _alias_keys(self, fact: AtomicFact, profile_id: str) -> set[str]:
+        out: set[str] = set()
+        for name in (getattr(fact, "canonical_entities", None) or []):
+            if not name:
+                continue
+            try:
+                ent = self._db.get_entity_by_name(name, profile_id)
+            except Exception:
+                ent = None
+            if ent is None:
+                continue
+            canonical = getattr(ent, "canonical_name", None)
+            if canonical:
+                out.add(canonical)
+            entity_id = getattr(ent, "entity_id", None)
+            if not entity_id:
+                continue
+            try:
+                for alias in self._db.get_aliases_for_entity(entity_id, profile_id):
+                    a = getattr(alias, "alias", None)
+                    if a:
+                        out.add(a)
+            except Exception:
+                continue
+        return out
+
+    # -- Mode B/C: LLM paraphrases (own prompt, fail-open) -------------------
+
+    def _llm_available(self) -> bool:
+        if self._llm is None:
+            return False
+        check = getattr(self._llm, "is_available", None)
+        try:
+            return bool(check()) if callable(check) else bool(check)
+        except Exception:
+            return False
+
+    def _llm_keys(self, fact: AtomicFact) -> set[str]:
+        content = getattr(fact, "content", "") or ""
+        if not content:
+            return set()
+        prompt = (
+            "Give 3-6 short alternative search terms (synonyms, aliases, or "
+            "paraphrases) that a person might use to look up the memory below. "
+            "Output ONLY a comma-separated list, no numbering, no explanation.\n\n"
+            f"Memory: {content}"
+        )
+        raw = self._invoke_llm(prompt)
+        if not raw:
+            return set()
+        seen: list[str] = []
+        for part in raw.replace("\n", ",").split(","):
+            p = part.strip()
+            if p and p not in seen:
+                seen.append(p)
+            if len(seen) >= _MAX_LLM_KEYS:
+                break
+        return set(seen)
+
+    def _invoke_llm(self, prompt: str) -> str:
+        """Call whatever generation method the injected LLM exposes; fail-open."""
+        for meth in ("generate", "complete", "chat"):
+            fn = getattr(self._llm, meth, None)
+            if callable(fn):
+                try:
+                    out = fn(prompt)
+                    return out if isinstance(out, str) else str(out or "")
+                except Exception as exc:
+                    logger.debug("KeyExpander LLM (%s) failed: %s", meth, exc)
+                    return ""
+        return ""

package/src/superlocalmemory/core/maintenance.py

@@ -127,6 +127,7 @@ def run_maintenance(
         "sheaf_checked": 0,
         "entity_summaries_consolidated": 0,  # V3.4.40
         "orphan_metadata_gc": 0,             # v3.6.4 (P1-3)
+        "expansion_backfilled": 0,           # T3b
     }
 
     # P1-3 (embeddings-vector-02): sweep orphaned embedding_metadata left by
@@ -141,6 +142,28 @@ def run_maintenance(
     if not facts:
         return counts
 
+    # T3b: backfill fact-expansion alt-keys (Mode A, entity-alias based) for
+    # facts stored before expansion existed. Bounded per run + skips already-
+    # populated and entity-less facts, so it converges without re-work churn.
+    try:
+        from superlocalmemory.core.key_expander import KeyExpander
+        populated = {
+            dict(r)["fact_id"]
+            for r in db.execute("SELECT DISTINCT fact_id FROM fact_expansion_fts")
+        }
+        expander = KeyExpander(db)
+        for f in facts:
+            if counts["expansion_backfilled"] >= 500:
+                break
+            if f.fact_id in populated or not f.canonical_entities:
+                continue
+            alt = expander.expand(f, profile_id, mode="a")
+            if alt:
+                db.upsert_fact_expansion(f.fact_id, alt)
+                counts["expansion_backfilled"] += 1
+    except Exception as exc:  # pragma: no cover — legacy DB / missing FTS
+        logger.debug("expansion backfill skipped: %s", exc)
+
     # 1a. Backfill: seed uninitialized facts with metadata-aware positions (B+C)
     if config.math.langevin_persist_positions:
         try:

package/src/superlocalmemory/core/modes.py

@@ -109,7 +109,7 @@ MODE_C = ModeCapabilities(
     data_stays_local=False,
     description=(
         "FULL POWER — UNRESTRICTED. Best embeddings (text-embedding-3-large, 3072-dim). "
-        "Best LLMs (GPT-5.2, Claude Opus). Agentic multi-round retrieval. "
+        "Best configured cloud LLMs (e.g. GPT-5, Claude Opus 4). Agentic multi-round retrieval. "
         "Cohere reranker option. No EU restriction. Target: 90%+"
     ),
 )

package/src/superlocalmemory/core/mutations.py

@@ -59,7 +59,7 @@ def delete_fact_authorized(
     if not rows:
         return {"ok": False, "error": f"Memory {fact_id} not found"}
     content_preview = dict(rows[0]).get("content", "")[:80]
-    engine._db.delete_fact(fact_id)
+    engine._db.delete_fact(fact_id, profile_id=profile_id)
     try:
         from superlocalmemory.core.backend_orchestrator import get_orchestrator
         orchestrator = get_orchestrator()
@@ -120,7 +120,7 @@ def update_fact_authorized(
                 updates["fisher_variance"] = fisher_variance
         except Exception as exc:
             logger.warning("UPDATE embedding refresh failed: %s", exc)
-    engine._db.update_fact(fact_id, updates)
+    engine._db.update_fact(fact_id, updates, profile_id=profile_id)
     try:
         from superlocalmemory.core.backend_orchestrator import get_orchestrator
         orchestrator = get_orchestrator()