superlocalmemory 3.7.8 → 3.8.1

package/src/superlocalmemory/core/graph_analyzer.py

@@ -211,6 +211,43 @@ class GraphAnalyzer:
                 result[node] = comm_id
         return result
 
+    def detect_communities_louvain(
+        self,
+        graph: Any = None,
+        profile_id: str = "",
+    ) -> dict[str, int]:
+        """Detect communities via Louvain (modularity-optimizing).
+
+        Higher quality than Label Propagation (deterministic with a seed,
+        no giant-community collapse), pure-Python via networkx — no extra
+        binary deps. Falls back to Label Propagation if unavailable.
+        """
+        import networkx as nx
+
+        if graph is None:
+            graph = self._build_networkx_graph(profile_id)
+        if graph.number_of_nodes() == 0:
+            return {}
+
+        undirected = graph.to_undirected()
+        try:
+            from networkx.algorithms.community import louvain_communities
+
+            communities = louvain_communities(
+                undirected, weight="weight", seed=42,
+            )
+        except Exception as exc:
+            logger.debug(
+                "Louvain unavailable/failed (%s); using Label Propagation", exc,
+            )
+            return self.detect_communities(graph, profile_id)
+
+        result: dict[str, int] = {}
+        for comm_id, community in enumerate(communities):
+            for node in community:
+                result[node] = comm_id
+        return result
+
     # ── v3.4.1: Leiden Community Detection ────────────────────────
 
     def detect_communities_leiden(
@@ -234,9 +271,9 @@ class GraphAnalyzer:
             import igraph
         except ImportError:
             logger.info(
-                "leidenalg not installed, using Label Propagation fallback",
+                "leidenalg not installed, using Louvain fallback",
             )
-            return self.detect_communities(graph, profile_id)
+            return self.detect_communities_louvain(graph, profile_id)
 
         # Convert DiGraph -> undirected -> igraph
         undirected = graph.to_undirected()

package/src/superlocalmemory/core/graph_pruner.py

@@ -350,8 +350,12 @@ def _cap_node_degree(
     Algorithm (single-pass window function — no Python loops):
       1. ROW_NUMBER() OVER (PARTITION BY source_id ORDER BY weight DESC) ranks
          every edge per node in one full table scan.
-      2. Edges with rn > max_degree are deleted in a single DELETE statement.
-      Requires SQLite 3.25+ (window functions). System is on 3.53.1.
+      2. Excess edge IDs are collected in a reusable temp table.
+      3. A single DELETE statement removes them; rowcount is returned.
+      4. The temp table is created once and cleared via DELETE (not DROP) —
+         DROP TABLE acquires an EXCLUSIVE lock that conflicts with concurrent
+         writers, causing "database is locked".  Using CREATE...IF NOT EXISTS
+         plus DELETE FROM avoids that conflict while preserving rowcount.
     """
     # gi-03: cap BOTH out-degree (PARTITION BY source_id) AND in-degree
     # (PARTITION BY target_id). Previously only out-degree was capped, so hub
@@ -379,9 +383,13 @@ def _cap_node_degree(
         )
         return excess
 
-    # Step 1: collect edges exceeding the cap in either direction (one pass).
-    c.execute("DROP TABLE IF EXISTS _slm_cap_del")
-    c.execute("CREATE TEMP TABLE _slm_cap_del (edge_id TEXT PRIMARY KEY)")
+    # CREATE IF NOT EXISTS + DELETE FROM instead of DROP + CREATE.
+    # DROP TABLE acquires EXCLUSIVE which conflicts with concurrent writers.
+    # CREATE IF NOT EXISTS is idempotent; DELETE FROM clears prior contents.
+    c.execute(
+        "CREATE TEMP TABLE IF NOT EXISTS _slm_cap_del (edge_id TEXT PRIMARY KEY)"
+    )
+    c.execute("DELETE FROM _slm_cap_del")
     c.execute(
         """
         INSERT OR IGNORE INTO _slm_cap_del (edge_id)
@@ -396,7 +404,6 @@ def _cap_node_degree(
         (profile_id, max_degree, max_degree),
     )
 
-    # Step 2: delete the over-cap edges (single DELETE).
     c.execute(
         """
         DELETE FROM graph_edges
@@ -407,8 +414,6 @@ def _cap_node_degree(
     )
     deleted = c.rowcount
 
-    c.execute("DROP TABLE IF EXISTS _slm_cap_del")
-
     logger.info(
         "_cap_node_degree: deleted %d low-weight edges (max_degree=%d, in+out capped)",
         deleted, max_degree,

package/src/superlocalmemory/core/ingestion_command.py

@@ -24,6 +24,7 @@ from typing import Any, Callable
 from superlocalmemory.storage.database import DatabaseManager
 
 _MATERIALIZATION_LOCKS = tuple(threading.RLock() for _ in range(64))
+_MAX_AUTOMATIC_MATERIALIZATION_ATTEMPTS = 10
 
 
 def _materialization_lock(operation_id: str) -> threading.RLock:
@@ -51,6 +52,10 @@ class OperationInProgress(RuntimeError):
     """Another live lease owner is materializing this operation."""
 
 
+class LeaseLost(OperationInProgress):
+    """The materializer no longer owns its durable operation lease."""
+
+
 def _canonical_json(value: Any) -> str:
     return json.dumps(value, sort_keys=True, separators=(",", ":"))
 
@@ -284,12 +289,19 @@ class IngestionOperationRepository:
             self._from_row(row)
             for row in self.db.execute(
                 "SELECT * FROM ingestion_operations "
-                "WHERE (state='queryable' AND created_at <= "
+                "WHERE attempt_count < ? AND ("
+                "(state='queryable' AND created_at <= "
                 "strftime('%Y-%m-%dT%H:%M:%fZ', 'now', ?)) "
                 "OR (state='failed' AND next_retry_at <= ?) "
-                "OR (state='enriching' AND lease_expires_at <= ?) "
+                "OR (state='enriching' AND lease_expires_at <= ?)) "
                 "ORDER BY created_at, rowid LIMIT ?",
-                (grace_modifier, now, now, max(1, int(limit))),
+                (
+                    _MAX_AUTOMATIC_MATERIALIZATION_ATTEMPTS,
+                    grace_modifier,
+                    now,
+                    now,
+                    max(1, int(limit)),
+                ),
             )
         ]
 
@@ -415,6 +427,27 @@ class IngestionOperationRepository:
         )
         return operation
 
+    def renew_enriching_lease(
+        self,
+        operation_id: str,
+        *,
+        owner: str,
+        lease_seconds: float,
+    ) -> bool:
+        """Extend a live lease only while the same owner still holds it."""
+        rows = self.db.execute(
+            "UPDATE ingestion_operations SET lease_expires_at=?, "
+            "updated_at=strftime('%Y-%m-%dT%H:%M:%fZ', 'now') "
+            "WHERE operation_id=? AND state='enriching' AND lease_owner=? "
+            "RETURNING operation_id",
+            (
+                time.time() + max(1.0, float(lease_seconds)),
+                operation_id,
+                owner,
+            ),
+        )
+        return bool(rows)
+
     def finish_enriching(
         self,
         operation_id: str,
@@ -469,6 +502,7 @@ class MaterializationResult:
 
     fact_ids: tuple[str, ...]
     derivation_state: dict[str, bool]
+    last_error: str = ""
 
 
 QueryableWriter = Callable[[IngestionRequest, str], list[str]]
@@ -500,6 +534,47 @@ class IngestionCommand:
         self._lease_seconds = max(1.0, float(lease_seconds))
         self._owner = f"ingestion-worker:{uuid.uuid4().hex}"
 
+    def _run_with_lease_heartbeat(
+        self,
+        operation_id: str,
+        callback: Callable[[], Any],
+    ) -> Any:
+        """Run slow work while periodically renewing its owner-bound lease."""
+        stop = threading.Event()
+        lost = threading.Event()
+        interval = min(30.0, max(0.1, self._lease_seconds / 3.0))
+
+        def heartbeat() -> None:
+            while not stop.wait(interval):
+                try:
+                    renewed = self.repository.renew_enriching_lease(
+                        operation_id,
+                        owner=self._owner,
+                        lease_seconds=self._lease_seconds,
+                    )
+                except sqlite3.Error:
+                    continue
+                if not renewed:
+                    lost.set()
+                    return
+
+        thread = threading.Thread(
+            target=heartbeat,
+            name=f"ingestion-lease-heartbeat:{operation_id[:8]}",
+            daemon=True,
+        )
+        thread.start()
+        try:
+            result = callback()
+        finally:
+            stop.set()
+            thread.join(timeout=max(1.0, interval * 2))
+        if lost.is_set():
+            raise LeaseLost(
+                f"ingestion lease lost for operation {operation_id}"
+            )
+        return result
+
     def submit(self, request: IngestionRequest) -> IngestionOperation:
         operation, _created = self.submit_with_status(request)
         return operation
@@ -548,27 +623,41 @@ class IngestionCommand:
         )
         if enriching.state is IngestionState.COMPLETE:
             return enriching
-        if enriching.final_fact_ids:
+        if (
+            enriching.final_fact_ids
+            and all(enriching.derivation_state.values())
+        ):
             return self._project_and_complete(enriching)
         try:
+            # Materialization is a durable saga, not one long SQLite
+            # transaction. Extractors, embedders, and local model calls can
+            # take minutes on a mature installation; keeping a write
+            # transaction open across that work blocks every interactive
+            # remember/update/delete and makes the dashboard appear dead.
+            #
+            # The materializer commits its relational checkpoints in short
+            # database operations. The operation lease and derivation state
+            # remain the recovery boundary, and only the final state-machine
+            # checkpoint is grouped atomically below.
+            materialized = self._run_with_lease_heartbeat(
+                operation_id,
+                lambda: self._materializer(enriching),
+            )
+            if isinstance(materialized, MaterializationResult):
+                fact_ids = tuple(materialized.fact_ids)
+                derivation_state = dict(materialized.derivation_state)
+                materialization_error = materialized.last_error
+            else:
+                fact_ids = tuple(materialized)
+                derivation_state = {"materializer": True}
+                materialization_error = ""
+            if not fact_ids:
+                raise RuntimeError("materialization produced no final facts")
+            incomplete = sorted(
+                name for name, complete in derivation_state.items()
+                if not complete
+            )
             with self.repository.db.transaction():
-                materialized = self._materializer(enriching)
-                if isinstance(materialized, MaterializationResult):
-                    fact_ids = tuple(materialized.fact_ids)
-                    derivation_state = dict(materialized.derivation_state)
-                else:
-                    fact_ids = tuple(materialized)
-                    derivation_state = {"materializer": True}
-                if not fact_ids:
-                    raise RuntimeError("materialization produced no final facts")
-                incomplete = sorted(
-                    name for name, complete in derivation_state.items()
-                    if not complete
-                )
-                if incomplete:
-                    raise RuntimeError(
-                        "incomplete derivation stages: " + ", ".join(incomplete)
-                    )
                 checkpointed = self.repository.checkpoint_enriching(
                     operation_id,
                     final_fact_ids=fact_ids,
@@ -577,6 +666,21 @@ class IngestionCommand:
                     lease_owner=self._owner,
                     lease_seconds=self._lease_seconds,
                 )
+            if materialization_error or incomplete:
+                error = materialization_error or (
+                    "incomplete derivation stages: " + ", ".join(incomplete)
+                )
+                return self.repository.finish_enriching(
+                    operation_id,
+                    owner=self._owner,
+                    target=IngestionState.FAILED,
+                    final_fact_ids=fact_ids,
+                    derivation_version=self._derivation_version,
+                    derivation_state=derivation_state,
+                    last_error=error,
+                )
+        except LeaseLost:
+            raise
         except Exception as exc:
             return self.repository.finish_enriching(
                 operation_id,
@@ -598,10 +702,13 @@ class IngestionCommand:
                 owner=self._owner,
                 lease_seconds=self._lease_seconds,
             )
-            projection_state = (
-                dict(self._projector(operation))
-                if self._projector is not None
-                else {}
+            projection_state = self._run_with_lease_heartbeat(
+                operation.operation_id,
+                lambda: (
+                    dict(self._projector(operation))
+                    if self._projector is not None
+                    else {}
+                ),
             )
             combined = {**operation.derivation_state, **projection_state}
             incomplete = sorted(
@@ -619,6 +726,8 @@ class IngestionCommand:
                 derivation_version=self._derivation_version,
                 derivation_state=combined,
             )
+        except LeaseLost:
+            raise
         except Exception as exc:
             return self.repository.finish_enriching(
                 operation.operation_id,

package/src/superlocalmemory/core/injection.py

@@ -128,20 +128,25 @@ def _load_injection_config():
 
 
 def estimate_tokens(text: str) -> int:
-    """chars/4 heuristic. Optional tiktoken if installed (best-effort).
-
-    tiktoken is an optional dependency (pip install superlocalmemory[injection]).
-    Falls back to chars/4 if tiktoken is not installed or raises any error.
-    All exception types (MemoryError, SystemError etc.) are subclasses of
-    Exception in Python ≥ 3.11, so the bare except Exception covers all paths.
+    """Estimate context tokens without loading native code by default.
+
+    The stable default is the deterministic ``chars / 4`` heuristic.  Loading
+    the optional native ``tiktoken`` extension in a long-lived daemon that also
+    hosts torch/scipy workers can terminate the whole process on a native fault;
+    Python exception handling cannot recover from SIGBUS/SIGSEGV.  Users who
+    explicitly need exact OpenAI token counts can opt in with
+    ``SLM_INJECTION_EXACT_TOKENS=1``.
     """
     if not text:
         return 0
+    heuristic = max(1, len(text) // 4)
+    if os.environ.get("SLM_INJECTION_EXACT_TOKENS", "0") != "1":
+        return heuristic
     try:
         import tiktoken
         return len(tiktoken.get_encoding("cl100k_base").encode(text))
     except Exception:
-        return max(1, len(text) // 4)
+        return heuristic
 
 
 def resolve_budget(mode: str, cfg) -> int:

package/src/superlocalmemory/core/key_expander.py

@@ -0,0 +1,138 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under AGPL-3.0-or-later - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""Fact-augmented key expansion (Phase 4, T3b).
+
+Generates *alternate keys* for a fact — synonyms, aliases, and paraphrases —
+that get indexed in ``fact_expansion_fts`` and UNION'd into BM25 retrieval, so a
+query for "automobile" or "the Big Apple" can match a fact that only says "car"
+or "NYC".
+
+Two tiers, matching the rest of SLM:
+  * Mode A (zero-LLM): pulls the fact's resolved entities' canonical names and
+    aliases from SLM's own entity graph — no model, no cost, and it reuses the
+    entity resolution already done at ingest.
+  * Mode B/C: additionally asks the LLM for a few paraphrase keywords (own
+    prompt, fail-open, bounded).
+
+Keys already present in the fact's content are dropped — indexing them twice
+adds nothing. Returns a single space-joined string ready for the FTS row.
+
+Part of Qualixar | Author: Varun Pratap Bhardwaj
+License: AGPL-3.0-or-later
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from superlocalmemory.storage.database import DatabaseManager
+    from superlocalmemory.storage.models import AtomicFact
+
+logger = logging.getLogger(__name__)
+
+# Bound the LLM enrichment so a pathological response can't bloat the index.
+_MAX_LLM_KEYS = 8
+_MAX_KEY_LEN = 60
+
+
+class KeyExpander:
+    """Produces alternate search keys for a fact (T3b)."""
+
+    __slots__ = ("_db", "_llm")
+
+    def __init__(self, db: DatabaseManager, llm: Any = None) -> None:
+        self._db = db
+        self._llm = llm
+
+    def expand(self, fact: AtomicFact, profile_id: str, mode: str = "a") -> str:
+        """Return space-joined alternate keys for ``fact`` (may be empty)."""
+        keys: set[str] = set()
+        keys |= self._alias_keys(fact, profile_id)
+        if mode in ("b", "c") and self._llm_available():
+            keys |= self._llm_keys(fact)
+
+        content_low = (getattr(fact, "content", "") or "").lower()
+        cleaned = {
+            k.strip() for k in keys
+            if k and k.strip() and len(k.strip()) <= _MAX_KEY_LEN
+            and k.strip().lower() not in content_low
+        }
+        return " ".join(sorted(cleaned))
+
+    # -- Mode A: entity aliases from SLM's own entity graph ------------------
+
+    def _alias_keys(self, fact: AtomicFact, profile_id: str) -> set[str]:
+        out: set[str] = set()
+        for name in (getattr(fact, "canonical_entities", None) or []):
+            if not name:
+                continue
+            try:
+                ent = self._db.get_entity_by_name(name, profile_id)
+            except Exception:
+                ent = None
+            if ent is None:
+                continue
+            canonical = getattr(ent, "canonical_name", None)
+            if canonical:
+                out.add(canonical)
+            entity_id = getattr(ent, "entity_id", None)
+            if not entity_id:
+                continue
+            try:
+                for alias in self._db.get_aliases_for_entity(entity_id, profile_id):
+                    a = getattr(alias, "alias", None)
+                    if a:
+                        out.add(a)
+            except Exception:
+                continue
+        return out
+
+    # -- Mode B/C: LLM paraphrases (own prompt, fail-open) -------------------
+
+    def _llm_available(self) -> bool:
+        if self._llm is None:
+            return False
+        check = getattr(self._llm, "is_available", None)
+        try:
+            return bool(check()) if callable(check) else bool(check)
+        except Exception:
+            return False
+
+    def _llm_keys(self, fact: AtomicFact) -> set[str]:
+        content = getattr(fact, "content", "") or ""
+        if not content:
+            return set()
+        prompt = (
+            "Give 3-6 short alternative search terms (synonyms, aliases, or "
+            "paraphrases) that a person might use to look up the memory below. "
+            "Output ONLY a comma-separated list, no numbering, no explanation.\n\n"
+            f"Memory: {content}"
+        )
+        raw = self._invoke_llm(prompt)
+        if not raw:
+            return set()
+        seen: list[str] = []
+        for part in raw.replace("\n", ",").split(","):
+            p = part.strip()
+            if p and p not in seen:
+                seen.append(p)
+            if len(seen) >= _MAX_LLM_KEYS:
+                break
+        return set(seen)
+
+    def _invoke_llm(self, prompt: str) -> str:
+        """Call whatever generation method the injected LLM exposes; fail-open."""
+        for meth in ("generate", "complete", "chat"):
+            fn = getattr(self._llm, meth, None)
+            if callable(fn):
+                try:
+                    out = fn(prompt)
+                    return out if isinstance(out, str) else str(out or "")
+                except Exception as exc:
+                    logger.debug("KeyExpander LLM (%s) failed: %s", meth, exc)
+                    return ""
+        return ""

package/src/superlocalmemory/core/maintenance.py

@@ -127,6 +127,7 @@ def run_maintenance(
         "sheaf_checked": 0,
         "entity_summaries_consolidated": 0,  # V3.4.40
         "orphan_metadata_gc": 0,             # v3.6.4 (P1-3)
+        "expansion_backfilled": 0,           # T3b
     }
 
     # P1-3 (embeddings-vector-02): sweep orphaned embedding_metadata left by
@@ -141,6 +142,28 @@ def run_maintenance(
     if not facts:
         return counts
 
+    # T3b: backfill fact-expansion alt-keys (Mode A, entity-alias based) for
+    # facts stored before expansion existed. Bounded per run + skips already-
+    # populated and entity-less facts, so it converges without re-work churn.
+    try:
+        from superlocalmemory.core.key_expander import KeyExpander
+        populated = {
+            dict(r)["fact_id"]
+            for r in db.execute("SELECT DISTINCT fact_id FROM fact_expansion_fts")
+        }
+        expander = KeyExpander(db)
+        for f in facts:
+            if counts["expansion_backfilled"] >= 500:
+                break
+            if f.fact_id in populated or not f.canonical_entities:
+                continue
+            alt = expander.expand(f, profile_id, mode="a")
+            if alt:
+                db.upsert_fact_expansion(f.fact_id, alt)
+                counts["expansion_backfilled"] += 1
+    except Exception as exc:  # pragma: no cover — legacy DB / missing FTS
+        logger.debug("expansion backfill skipped: %s", exc)
+
     # 1a. Backfill: seed uninitialized facts with metadata-aware positions (B+C)
     if config.math.langevin_persist_positions:
         try:

package/src/superlocalmemory/core/maintenance_scheduler.py

@@ -8,7 +8,8 @@ V3.3.13: Periodically triggers Langevin/Ebbinghaus/Sheaf maintenance
 so users don't need to call run_maintenance manually.
 
 Configurable interval via ForgettingConfig.scheduler_interval_minutes.
-Defaults to 30 min. Disabled during benchmarks (no config.forgetting.enabled).
+Defaults to 30 min. Optional forgetting/math work follows
+``config.forgetting.enabled``; tier evaluation and bounded housekeeping do not.
 
 Part of Qualixar | Author: Varun Pratap Bhardwaj
 License: AGPL-3.0-or-later
@@ -79,12 +80,21 @@ class MaintenanceScheduler:
         if not self._running:
             return
         for profile_id in self._profile_ids():
-            try:
-                from superlocalmemory.core.maintenance import run_maintenance
-                counts = run_maintenance(self._db, self._config, profile_id)
-                logger.info("Scheduled maintenance complete for %s: %s", profile_id, counts)
-            except Exception as exc:
-                logger.warning("Scheduled maintenance failed for %s: %s", profile_id, exc)
+            if self._config.forgetting.enabled:
+                try:
+                    from superlocalmemory.core.maintenance import run_maintenance
+                    counts = run_maintenance(self._db, self._config, profile_id)
+                    logger.info(
+                        "Scheduled maintenance complete for %s: %s",
+                        profile_id,
+                        counts,
+                    )
+                except Exception as exc:
+                    logger.warning(
+                        "Scheduled maintenance failed for %s: %s",
+                        profile_id,
+                        exc,
+                    )
 
             # V3.4.11: Graph pruning (remove orphan edges)
             try:

package/src/superlocalmemory/core/modes.py

@@ -109,7 +109,7 @@ MODE_C = ModeCapabilities(
     data_stays_local=False,
     description=(
         "FULL POWER — UNRESTRICTED. Best embeddings (text-embedding-3-large, 3072-dim). "
-        "Best LLMs (GPT-5.2, Claude Opus). Agentic multi-round retrieval. "
+        "Best configured cloud LLMs (e.g. GPT-5, Claude Opus 4). Agentic multi-round retrieval. "
         "Cohere reranker option. No EU restriction. Target: 90%+"
     ),
 )

package/src/superlocalmemory/core/mutations.py

@@ -59,7 +59,7 @@ def delete_fact_authorized(
     if not rows:
         return {"ok": False, "error": f"Memory {fact_id} not found"}
     content_preview = dict(rows[0]).get("content", "")[:80]
-    engine._db.delete_fact(fact_id)
+    engine._db.delete_fact(fact_id, profile_id=profile_id)
     try:
         from superlocalmemory.core.backend_orchestrator import get_orchestrator
         orchestrator = get_orchestrator()
@@ -120,7 +120,7 @@ def update_fact_authorized(
                 updates["fisher_variance"] = fisher_variance
         except Exception as exc:
             logger.warning("UPDATE embedding refresh failed: %s", exc)
-    engine._db.update_fact(fact_id, updates)
+    engine._db.update_fact(fact_id, updates, profile_id=profile_id)
     try:
         from superlocalmemory.core.backend_orchestrator import get_orchestrator
         orchestrator = get_orchestrator()