superlocalmemory 3.8.1 → 3.8.2

package/src/superlocalmemory/core/maintenance.py

@@ -109,6 +109,7 @@ def run_maintenance(
     db: DatabaseManager,
     config: SLMConfig,
     profile_id: str = "default",
+    embedder: object | None = None,
 ) -> dict[str, int]:
     """Run background maintenance on mathematical layers.
 
@@ -116,6 +117,11 @@ def run_maintenance(
         db: Database manager.
         config: Full SLM configuration.
         profile_id: Scope to this profile.
+        embedder: Optional embedder for self-healing NULL-embedding backfill.
+            When provided and NULL embeddings exist, up to 100 facts are
+            embedded per maintenance pass so the DB converges over time.
+            Pass ``None`` (default) to skip the backfill — existing callers
+            are unaffected.
 
     Returns:
         Dict of counts: langevin_updated, sheaf_checked, etc.
@@ -128,6 +134,7 @@ def run_maintenance(
         "entity_summaries_consolidated": 0,  # V3.4.40
         "orphan_metadata_gc": 0,             # v3.6.4 (P1-3)
         "expansion_backfilled": 0,           # T3b
+        "embeddings_backfilled": 0,          # v3.8.x NULL-embedding self-heal
     }
 
     # P1-3 (embeddings-vector-02): sweep orphaned embedding_metadata left by
@@ -138,6 +145,42 @@ def run_maintenance(
     except Exception as exc:  # pragma: no cover - defensive
         logger.debug("orphan metadata GC skipped: %s", exc)
 
+    # v3.8.x: self-healing NULL-embedding backfill.  Facts stored while the
+    # embedder was unavailable end up with NULL embedding and are invisible to
+    # semantic recall.  When an embedder is available, embed up to 100 facts
+    # per maintenance pass so the DB converges without blocking the caller.
+    if embedder is not None:
+        try:
+            from superlocalmemory.storage.embedding_migrator import (
+                backfill_missing_embeddings,
+            )
+
+            # Guard: skip entirely when nothing needs backfilling.
+            null_rows = db.execute(
+                "SELECT count(*) AS c FROM atomic_facts "
+                "WHERE embedding IS NULL AND profile_id = ?",
+                (profile_id,),
+            )
+            null_count = int(null_rows[0]["c"]) if null_rows else 0
+            if null_count > 0:
+                result = backfill_missing_embeddings(
+                    config,
+                    db,
+                    embedder,
+                    batch_size=50,
+                    limit=100,
+                )
+                counts["embeddings_backfilled"] = result["embedded"]
+                if result["embedded"] > 0:
+                    logger.info(
+                        "Maintenance embedding backfill: %d facts embedded, "
+                        "%d remaining.",
+                        result["embedded"],
+                        result["remaining_null"],
+                    )
+        except Exception as exc:
+            logger.debug("embedding backfill skipped during maintenance: %s", exc)
+
     facts = db.get_all_facts(profile_id)
     if not facts:
         return counts

package/src/superlocalmemory/core/maintenance_scheduler.py

@@ -40,10 +40,16 @@ class MaintenanceScheduler:
         db: DatabaseManager,
         config: SLMConfig,
         profile_id: str = "default",
+        embedder: object | None = None,
     ) -> None:
         self._db = db
         self._config = config
         self._profile_id = profile_id
+        # v3.8.2 self-heal: when provided, periodic maintenance backfills
+        # NULL embeddings so a DB stays fully queryable over time even if
+        # facts were stored while the embedder was unavailable. Runs
+        # independently of forgetting.enabled (see _run).
+        self._embedder = embedder
         self._timer: threading.Timer | None = None
         self._running = False
         self._interval = config.forgetting.scheduler_interval_minutes * 60.0
@@ -79,6 +85,28 @@ class MaintenanceScheduler:
         """Execute maintenance + auto-backup check, then schedule next run."""
         if not self._running:
             return
+        # v3.8.2 self-heal: bounded NULL-embedding backfill runs every cycle
+        # INDEPENDENTLY of forgetting.enabled and across ALL profiles — a fact
+        # stored while the embedder was down must become queryable again without
+        # the user touching anything. Idempotent + bounded (200/pass) so it
+        # converges quietly and is a no-op once coverage is complete.
+        if self._embedder is not None:
+            try:
+                from superlocalmemory.storage.embedding_migrator import (
+                    backfill_missing_embeddings,
+                )
+                r = backfill_missing_embeddings(
+                    self._config, self._db, self._embedder,
+                    limit=50, all_profiles=True,
+                )
+                if r.get("embedded"):
+                    logger.info(
+                        "Self-heal backfill: %d embedded, %d remaining",
+                        r["embedded"], r["remaining_null"],
+                    )
+            except Exception as exc:
+                logger.debug("Self-heal backfill skipped: %s", exc)
+
         for profile_id in self._profile_ids():
             if self._config.forgetting.enabled:
                 try:

package/src/superlocalmemory/core/recall_pipeline.py

@@ -604,6 +604,33 @@ def apply_v2_bandit_ensemble(
 # run_recall  (was MemoryEngine.recall)
 # ---------------------------------------------------------------------------
 
+def resolve_hot_path_fast(fast: bool | None, config: "SLMConfig") -> bool:
+    """Resolve the recall ``fast`` flag when a caller leaves it unset (None).
+
+    v3.8.2 client-driven agentic: the agent hot path (CLI / MCP / plugins) is
+    consumed by a frontier LLM (Claude Code, Copilot, Codex, …) that reformulates
+    multi-hop / low-confidence queries far better than the local Ollama model.
+    So an unset ``fast`` defaults to True — skip the internal agentic round and
+    let the calling LLM drive refinement — whenever ``retrieval.client_driven_agentic``
+    is on (the ship default). An explicit ``True``/``False`` from the caller
+    always wins (the dashboard search path passes ``True`` for a snappy list;
+    a no-smart-client deployment can pass ``False``). Env override
+    ``SLM_HOT_PATH_INTERNAL_AGENTIC=1`` forces internal-agentic-on globally.
+
+    This is the single resolution point: every recall path (HTTP, MCP, CLI,
+    in-process adapter) funnels through ``run_recall`` and calls this, so the
+    client-driven default is consistent everywhere by construction.
+    """
+    if fast is not None:
+        return bool(fast)
+    import os
+    rc = getattr(config, "retrieval", None)
+    client_driven = bool(getattr(rc, "client_driven_agentic", True))
+    if os.environ.get("SLM_HOT_PATH_INTERNAL_AGENTIC") == "1":
+        client_driven = False
+    return client_driven
+
+
 def run_recall(
     query: str,
     profile_id: str,
@@ -620,7 +647,7 @@ def run_recall(
     hooks: HookRegistry,
     access_log: Any = None,
     auto_linker: Any = None,
-    fast: bool = False,
+    fast: bool | None = None,
     include_global: bool = False,
     include_shared: bool = False,
     window: str | tuple[str, str] | None = None,
@@ -632,8 +659,11 @@ def run_recall(
 
     Pipeline: retrieval -> agentic sufficiency (if configured) -> post-recall updates.
 
-    ``fast=True`` skips remote agentic verification while retaining the six
-    local retrieval channels.
+    ``fast=True`` skips the internal agentic verification round while retaining
+    the six local retrieval channels + reranker. ``fast=None`` (unset) resolves
+    to the client-driven-agentic default (see ``resolve_hot_path_fast``): the
+    agent hot path skips the internal round and delegates refinement to the
+    calling LLM. ``fast=False`` forces the internal agentic round.
     """
     # Pre-operation hooks
     hook_ctx = {
@@ -646,6 +676,12 @@ def run_recall(
 
     m = mode or config.mode
 
+    # v3.8.2: resolve the client-driven-agentic default when a caller left
+    # ``fast`` unset (None). After this line ``fast`` is a concrete bool, so
+    # the agentic gate below (``if not fast``) behaves identically for every
+    # entry point that funnels through here.
+    fast = resolve_hot_path_fast(fast, config)
+
     # v3.5.0 diagnostic: per-stage recall timing under SLM_RECALL_TIMING=1.
     # Zero overhead when the env var is unset. Permanent observability hook.
     import os as _os_t

package/src/superlocalmemory/core/store_pipeline.py

@@ -89,6 +89,48 @@ def _record_fact_entity_association(
                 "WHERE entity_id=? AND profile_id=?",
                 (entity_id, profile_id),
             )
+        elif not claimed:
+            # An empty result has TWO causes: (a) the 'historical-backfill'
+            # repair-state row is missing — M028 DDL ran but its INSERT rolled
+            # back (partial migration) — so the JOIN returned zero rows and the
+            # main INSERT produced no output; or (b) the association already
+            # exists and the ON CONFLICT update was correctly skipped (an
+            # idempotent re-run). Only (a) needs the fallback insert; (b) is a
+            # no-op. Disambiguate by probing the repair-state table directly —
+            # a DIFFERENT table, so this stays a constant-query, idempotent
+            # effect and never re-fires the insert on an already-applied row.
+            repair_ready = db.execute(
+                "SELECT 1 FROM fact_entity_association_repair_state "
+                "WHERE repair_key='historical-backfill' LIMIT 1",
+            )
+            if repair_ready:
+                # (b) idempotent skip — the association was already applied.
+                return
+            # (a) partial migration: insert with count_applied=1 (no historical-
+            # rowid check needed — all associations in this state are
+            # post-migration). ON CONFLICT DO NOTHING makes this retry-safe.
+            fallback = db.execute(
+                "INSERT INTO fact_entity_associations "
+                "(profile_id,fact_id,entity_id,first_operation_id,count_applied) "
+                "SELECT ?,?,?,?,1 "
+                "FROM canonical_entities AS entity "
+                "JOIN atomic_facts AS fact "
+                "ON fact.fact_id=? AND fact.profile_id=? "
+                "WHERE entity.entity_id=? AND entity.profile_id=? "
+                "ON CONFLICT(profile_id,fact_id,entity_id) DO NOTHING "
+                "RETURNING count_applied",
+                (
+                    profile_id, fact_id, entity_id, operation_id,
+                    fact_id, profile_id,
+                    entity_id, profile_id,
+                ),
+            )
+            if fallback:
+                db.execute(
+                    "UPDATE canonical_entities SET fact_count=fact_count+1 "
+                    "WHERE entity_id=? AND profile_id=?",
+                    (entity_id, profile_id),
+                )
 
 
 def _init_langevin_position(dim: int = 8) -> list[float]:

package/src/superlocalmemory/mcp/_daemon_proxy.py

@@ -40,7 +40,7 @@ class DaemonPoolProxy:
 
     def recall(
         self, query: str, limit: int = 10, session_id: str = "",
-        fast: bool = False,
+        fast: bool | None = None,
         include_global: bool | None = None,
         include_shared: bool | None = None,
         window: str | None = None,
@@ -49,8 +49,12 @@ class DaemonPoolProxy:
             "q": query,
             "limit": limit,
             "session_id": session_id or "",
-            "fast": "true" if fast else "false",
         }
+        # v3.8.2 client-driven agentic: only send ``fast`` when the caller set it
+        # explicitly. Unset (None) lets the daemon resolve the configured
+        # client-driven-agentic default — the same way scope flags are handled.
+        if fast is not None:
+            _params["fast"] = "true" if fast else "false"
         # v3.6.15 multi-scope: only send the scope flags when explicitly set, so
         # an unset value lets the daemon resolve the configured default (shared
         # is opt-in). "None" must NOT become the string "none" on the wire.

package/src/superlocalmemory/mcp/_pool_adapter.py

@@ -98,7 +98,10 @@ def pool_recall(query: str, limit: int = 10, **kwargs: Any) -> PoolRecallRespons
         "query": query,
         "limit": limit,
         "session_id": str(kwargs.get("session_id") or ""),
-        "fast": bool(kwargs.get("fast", False)),
+        # v3.8.2 client-driven agentic: pass ``fast`` through unchanged. None
+        # (unset) flows to the daemon/engine which resolves the configured
+        # client-driven-agentic default; an explicit bool always wins.
+        "fast": kwargs.get("fast", None),
     }
     if "include_global" in kwargs:
         _recall_kwargs["include_global"] = kwargs["include_global"]

package/src/superlocalmemory/mcp/tools_active.py

@@ -227,7 +227,7 @@ def register_active_tools(server, get_engine: Callable) -> None:
                 # thread so the async MCP event loop is not stalled — same
                 # fix class as #34 mesh tools deadlock.
                 response = await asyncio.to_thread(
-                    pool_recall, search_query, limit=max_results, fast=False,
+                    pool_recall, search_query, limit=max_results, fast=None,
                 )
             except (PoolError, Exception) as exc:
                 logger.warning(

package/src/superlocalmemory/mcp/tools_core.py

@@ -282,13 +282,28 @@ def register_core_tools(server, get_engine: Callable) -> None:
     @server.tool(annotations=ToolAnnotations(readOnlyHint=True))
     async def recall(
         query: str, limit: int = CANONICAL_RECALL_LIMIT, agent_id: str = "mcp_client",
-        session_id: str = "", fast: bool = False,
+        session_id: str = "", fast: bool | None = None,
         include_global: bool | None = None,
         include_shared: bool | None = None,
         window: str = "",
     ) -> dict:
         """Search memories through hybrid retrieval, RRF fusion, and reranking.
 
+        Fast local retrieval (six channels + reranker) returns in ~1-2s. This
+        tool does NOT run an internal LLM reformulation round — YOU (the calling
+        model) are the reasoner. Drive refinement using the confidence signals
+        in the response:
+          • ``no_confident_match: true`` → nothing cleared the evidence floor.
+            Do NOT invent a memory. Rewrite the query into 1-3 more specific
+            sub-queries (split multi-hop questions; try entity names, synonyms,
+            or a broader phrasing) and call ``recall`` again before concluding
+            the information is unknown.
+          • ``answer_confidence`` low / ``abstained: true`` → the top hit is
+            weak. Re-query with a sharper phrasing, or widen with
+            ``include_shared=true`` / ``include_global=true`` if appropriate.
+          • Confident match → use it directly; no second call needed.
+        One extra targeted recall is cheap and beats a wrong "not found".
+
         Optional ``session_id`` threads through to the
         engine's outcome-queue so PostToolUse / Stop hooks can attach
         engagement signals to this recall. Claude Code should pass its
@@ -357,7 +372,7 @@ def register_core_tools(server, get_engine: Callable) -> None:
             # block behind a single threading.Lock. See worker_pool.py.
             result = await asyncio.to_thread(
                 pool.recall, query, limit=limit, session_id=effective_sid,
-                fast=bool(fast),
+                fast=fast,
                 include_global=include_global,
                 include_shared=include_shared,
                 window=window or None,

package/src/superlocalmemory/retrieval/bridge_discovery.py

@@ -27,6 +27,7 @@ License: AGPL-3.0-or-later
 from __future__ import annotations
 
 import logging
+import time
 from typing import TYPE_CHECKING
 
 from superlocalmemory.retrieval.scope_policy import (
@@ -71,6 +72,7 @@ class BridgeDiscovery:
         *,
         include_global: bool = False,
         include_shared: bool = False,
+        time_budget_s: float = 0.4,
     ) -> list[tuple[str, float]]:
         """Find bridge facts connecting seed results.
 
@@ -110,9 +112,19 @@ class BridgeDiscovery:
 
         bridges: list[tuple[str, float]] = []
         seen = set(seed_ids)
+        # v3.8.2: bound the per-entity get_facts_by_entity fan-out. On a dense
+        # entity graph (M5: 3.3k entities / 208k edges) a single recall could
+        # issue 200+ DB round-trips here — the primary 3.8 full-mode latency
+        # spike (observed 7.6s). Bridges are a SUPPLEMENTARY post-fusion boost
+        # (score x0.8, only added if not already found), so truncating them
+        # under a wall-clock budget is quality-safe: the ranked channels have
+        # already returned the core results.
+        deadline = time.monotonic() + time_budget_s
 
         # Check consecutive pairs for entity overlap
         for i in range(len(seed_ids) - 1):
+            if time.monotonic() > deadline:
+                break
             fact_a = seed_facts.get(seed_ids[i])
             fact_b = seed_facts.get(seed_ids[i + 1])
             if not fact_a or not fact_b:
@@ -128,6 +140,8 @@ class BridgeDiscovery:
             # Strategy 1: Entity bridge (union minus intersection)
             bridge_entities = (entities_a | entities_b) - (entities_a & entities_b)
             for eid in bridge_entities:
+                if time.monotonic() > deadline:
+                    break
                 entity_facts = self._db.get_facts_by_entity(
                     eid,
                     profile_id,

package/src/superlocalmemory/retrieval/spreading_activation.py

@@ -139,14 +139,19 @@ class SpreadingActivation:
             )
             # Owner-partitioned vector indexes cannot discover opted-in peers.
             # Add visible external embeddings with the same cosine seed signal.
-            try:
-                external_facts = self._db.get_external_visible_facts(
-                    profile_id,
-                    include_global=include_global,
-                    include_shared=include_shared,
-                )
-            except Exception:
-                external_facts = []
+            # v3.8.2 perf: external (global/shared) facts only matter for a
+            # cross-scope read. For the default personal scope this query always
+            # returns [] — skip it to remove a per-recall DB round-trip.
+            external_facts: list = []
+            if include_global or include_shared:
+                try:
+                    external_facts = self._db.get_external_visible_facts(
+                        profile_id,
+                        include_global=include_global,
+                        include_shared=include_shared,
+                    )
+                except Exception:
+                    external_facts = []
             q_vec = np.array(query, dtype=np.float32)
             q_norm = float(np.linalg.norm(q_vec))
             combined = {fact_id: score for fact_id, score in seed_results}
@@ -162,18 +167,26 @@ class SpreadingActivation:
                     continue
                 score = (float(np.dot(q_vec, fact_vec) / denominator) + 1.0) / 2.0
                 combined[fact.fact_id] = max(combined.get(fact.fact_id, 0.0), score)
-            allowed_seeds = authorized_fact_ids(
-                self._db,
-                combined,
-                profile_id,
-                include_global=include_global,
-                include_shared=include_shared,
-            )
-            seed_results = [
-                (fact_id, score)
-                for fact_id, score in combined.items()
-                if fact_id in allowed_seeds
-            ]
+            # v3.8.2 perf: seeds come from this profile's own vector index /
+            # get_all_facts(profile_id), so for personal scope they are already
+            # authorized. Only re-authorize when a cross-scope read merged in
+            # global/shared candidates. filter_authorized_results below remains the
+            # security net on the returned set.
+            if include_global or include_shared:
+                allowed_seeds = authorized_fact_ids(
+                    self._db,
+                    combined,
+                    profile_id,
+                    include_global=include_global,
+                    include_shared=include_shared,
+                )
+                seed_results = [
+                    (fact_id, score)
+                    for fact_id, score in combined.items()
+                    if fact_id in allowed_seeds
+                ]
+            else:
+                seed_results = list(combined.items())
             if not seed_results:
                 return []
 
@@ -186,13 +199,18 @@ class SpreadingActivation:
             )
             cached = self._get_cached_results(query_hash, profile_id)
             if cached:
-                return filter_authorized_results(
-                    self._db,
-                    cached,
-                    profile_id,
-                    include_global=include_global,
-                    include_shared=include_shared,
-                )[:top_k]
+                # v3.8.2 perf: cached activations were produced from this profile's
+                # own propagation; personal-scope hits need no re-authorization.
+                # Cross-scope hits still pass the fail-closed filter.
+                if include_global or include_shared:
+                    return filter_authorized_results(
+                        self._db,
+                        cached,
+                        profile_id,
+                        include_global=include_global,
+                        include_shared=include_shared,
+                    )[:top_k]
+                return cached[:top_k]
 
             # Run 5-step spreading activation
             activations = self._propagate(
@@ -312,7 +330,15 @@ class SpreadingActivation:
                 if activation < 0.001:
                     continue
 
-                # Get neighbors from BOTH tables (Rule 13) — cached per node
+                # Get neighbors from BOTH tables (Rule 13) — cached per node.
+                # v3.8.2 perf-fix: _get_unified_neighbors already filters edges by
+                # the scope predicate (graph_edges via _scope_where; association_edges
+                # by profile_id), so for the default personal scope the returned
+                # neighbors are inherently authorized. The per-node re-authorization
+                # (2 DB round-trips/node, ~30–60 per recall — the primary 3.8 latency
+                # regression) is only required when a cross-scope read can surface
+                # global/shared neighbors. filter_authorized_results() on the returned
+                # set remains the security net for every scope.
                 if node_id not in neighbor_cache:
                     raw_neighbors = self._get_unified_neighbors(
                         node_id,
@@ -320,16 +346,20 @@ class SpreadingActivation:
                         include_global=include_global,
                         include_shared=include_shared,
                     )
-                    allowed_neighbors = authorized_fact_ids(
-                        self._db,
-                        (neighbor_id for neighbor_id, _weight in raw_neighbors),
-                        profile_id,
-                        include_global=include_global,
-                        include_shared=include_shared,
-                    )
-                    neighbor_cache[node_id] = [
-                        item for item in raw_neighbors if item[0] in allowed_neighbors
-                    ]
+                    if include_global or include_shared:
+                        allowed_neighbors = authorized_fact_ids(
+                            self._db,
+                            (neighbor_id for neighbor_id, _weight in raw_neighbors),
+                            profile_id,
+                            include_global=include_global,
+                            include_shared=include_shared,
+                        )
+                        neighbor_cache[node_id] = [
+                            item for item in raw_neighbors
+                            if item[0] in allowed_neighbors
+                        ]
+                    else:
+                        neighbor_cache[node_id] = raw_neighbors
                 neighbors = neighbor_cache[node_id]
 
                 # Out-degree for fan effect normalization

package/src/superlocalmemory/server/routes/behavioral.py

@@ -473,8 +473,12 @@ def get_assertions(
         query += " ORDER BY confidence DESC LIMIT ?"
         params.append(limit)
 
-        rows = conn.execute(query, tuple(params)).fetchall()
-        conn.close()
+        # F8 fix: use try/finally so conn.close() is guaranteed even when
+        # conn.execute() raises (e.g. SQLITE_BUSY under dashboard burst load).
+        try:
+            rows = conn.execute(query, tuple(params)).fetchall()
+        finally:
+            conn.close()
 
         assertions = [dict(r) for r in rows]
         return {

package/src/superlocalmemory/server/routes/learning.py

@@ -36,6 +36,12 @@ router = APIRouter()
 
 LEARNING_DB = MEMORY_DIR / "learning.db"
 
+# F10 fix: track which profiles have completed their initial cache-bypass check
+# per daemon start.  First call uses use_cache=False (tamper detection);
+# subsequent calls use use_cache=True so tab navigation does not re-deserialize
+# a LightGBM model on every dashboard request.
+_ranker_initial_check_done: set[str] = set()
+
 
 def _require_write(request: Request) -> None:
     from superlocalmemory.access.rbac import Permission
@@ -137,10 +143,14 @@ def _compute_ranker_phase(
         )
 
     try:
-        # Official train/promote/rollback paths invalidate this cache.  Reuse
-        # the verified object here so tab navigation does not deserialize a
+        # F10 fix: first call per daemon start per profile must bypass the cache
+        # so a tampered model file is detected before being served.  Subsequent
+        # calls reuse the cache so tab navigation does not re-deserialize a
         # LightGBM model on every dashboard request.
-        model = load_active(db, profile_id, use_cache=True)
+        bypass_cache = profile_id not in _ranker_initial_check_done
+        if bypass_cache:
+            _ranker_initial_check_done.add(profile_id)
+        model = load_active(db, profile_id, use_cache=not bypass_cache)
     except Exception as exc:
         logger.warning("load_active failed: %s", exc)
         model = None

package/src/superlocalmemory/server/routes/memories.py

@@ -507,9 +507,14 @@ async def search_memories(request: Request, body: SearchRequest):
         # a stalled connection and aborts with "signal is aborted without reason"
         # before the response arrives. Fix: run in a thread-pool executor so the
         # event loop stays alive to send keepalive frames.
-        # v3.4.64 (regression fix): fast=True skips spreading_activation + agentic
-        # LLM rounds (saves ~7s on cold graph traversal). fast=False is the SLOW
-        # path that enables both; the earlier comment had this inverted.
+        # v3.8.2: fast=True — the dashboard search BOX is a snappy retrieval
+        # list (all six local channels + reranker), never the internal agentic
+        # LLM round, which would reintroduce the multi-second hang this endpoint
+        # is regression-tested against (test_search_fast_param_and_profile_isolation).
+        # The human-facing LLM synthesis lives on separate paths that are NOT the
+        # search list: the "ask" memory-chat (/api/v3/chat/stream, Ollama Mode B)
+        # and the precomputed knowledge-cluster summaries (core.community_summary,
+        # Mode B/C). So search stays fast; synthesis is where the LLM adds value.
         import asyncio
         import time as _time
         engine = _get_engine(request)