superlocalmemory 3.8.0 → 3.8.2

package/src/superlocalmemory/server/unified_daemon.py

@@ -31,6 +31,7 @@ import json
 import logging
 import os
 import signal
+import sqlite3
 import sys
 import threading
 import time
@@ -65,12 +66,51 @@ from superlocalmemory.infra.data_root import (
     canonical_data_root,
     state_path,
 )
+from superlocalmemory.learning.source_quality import (
+    SourceQualityRepairUnavailable,
+    enumerate_source_quality_repair_profiles,
+    repair_historical_source_quality,
+)
 
 logger = logging.getLogger("superlocalmemory.unified_daemon")
 
 _DEFAULT_PORT = 8765
 _LEGACY_PORT = 8767
 _ACTIVE_DAEMON_DESCRIPTOR: DaemonDescriptor | None = None
+_SOURCE_QUALITY_MAX_BATCH_SIZE = 250
+_SOURCE_QUALITY_PROFILE_REFRESH_SECONDS = 60.0
+_FACT_ENTITY_REPAIR_MIN_RETRY_SECONDS = 0.05
+_FACT_ENTITY_REPAIR_MAX_RETRY_SECONDS = 30.0
+# ``wait=true`` is a compatibility affordance, never permission to hold the
+# ASGI event loop hostage to a local LLM.  Normal clients omit it and receive
+# an immediate durable/queryable receipt; explicit waiters get this small
+# completion window, then the M018 materializer continues in the background.
+_REMEMBER_ENRICHMENT_WAIT_SECONDS = 0.75
+_SENSITIVE_READ_PREFIXES = (
+    "/api/memories", "/api/facts", "/api/clusters", "/api/graph",
+    "/api/v3/associations", "/api/v3/core-memory",
+    "/api/v3/soft-prompts", "/api/v3/dashboard", "/api/v3/mode",
+    "/api/v3/embedding/config", "/api/v3/scope/config",
+    "/api/v3/storage/config", "/api/v3/daemon/config",
+    "/api/v3/mesh/config", "/api/v3/trust/config",
+    "/api/v3/forgetting/config", "/api/v3/mcp/profiles",
+    "/api/learning", "/api/behavioral",
+)
+_SENSITIVE_READ_EXACT_PATHS = (
+    "/api/search", "/api/v3/recall/trace", "/api/patterns",
+    "/api/feedback/stats", "/api/stats", "/api/timeline",
+)
+
+
+def _is_sensitive_dashboard_read(method: str, path: str) -> bool:
+    return (
+        method == "GET"
+        and (
+            path.startswith(_SENSITIVE_READ_PREFIXES)
+            or path in _SENSITIVE_READ_EXACT_PATHS
+            or path.startswith("/api/v3/recall")
+        )
+    )
 
 
 def _rbac_read_gate(request, app_state):
@@ -715,6 +755,362 @@ def _warm_spreading_activation(engine, runtime) -> bool:
         return False
 
 
+def _set_source_quality_repair_status(application, **updates) -> dict:
+    current = getattr(
+        application.state, "source_quality_repair_status", {},
+    )
+    status = {**current, **updates}
+    application.state.source_quality_repair_status = status
+    return status
+
+
+def _schedule_fact_entity_association_repair(
+    application,
+    memory_db_path: Path,
+    *,
+    batch_size: int = 250,
+    tick_seconds: float = 1.0,
+) -> asyncio.Task:
+    """Schedule bounded M028 backfill only after readiness is published."""
+    from superlocalmemory.storage.migrations.M028_fact_entity_associations import (
+        get_repair_status,
+    )
+
+    durable = get_repair_status(Path(memory_db_path))
+    application.state.fact_entity_association_repair_status = {
+        **durable,
+        "source": "startup_background_repair",
+        "batch_size": batch_size,
+    }
+    task = asyncio.create_task(
+        _fact_entity_association_repair_loop(
+            application,
+            Path(memory_db_path),
+            batch_size=batch_size,
+            tick_seconds=tick_seconds,
+        ),
+        name="fact-entity-association-upgrade-repair",
+    )
+    application.state.fact_entity_association_repair_task = task
+    return task
+
+
+async def _fact_entity_association_repair_loop(
+    application,
+    memory_db_path: Path,
+    *,
+    batch_size: int,
+    tick_seconds: float,
+) -> None:
+    from superlocalmemory.storage.migrations.M028_fact_entity_associations import (
+        get_repair_status,
+        repair_fact_entity_associations,
+    )
+
+    consecutive_failures = 0
+    try:
+        while True:
+            try:
+                await asyncio.to_thread(
+                    repair_fact_entity_associations,
+                    memory_db_path,
+                    batch_size=batch_size,
+                    max_batches=1,
+                )
+                durable = await asyncio.to_thread(
+                    get_repair_status, memory_db_path,
+                )
+            except sqlite3.Error as exc:
+                consecutive_failures += 1
+                retry_delay = min(
+                    _FACT_ENTITY_REPAIR_MAX_RETRY_SECONDS,
+                    max(
+                        _FACT_ENTITY_REPAIR_MIN_RETRY_SECONDS,
+                        float(tick_seconds),
+                    ) * (2 ** min(consecutive_failures - 1, 10)),
+                )
+                try:
+                    durable = await asyncio.to_thread(
+                        get_repair_status, memory_db_path,
+                    )
+                except sqlite3.Error:
+                    durable = getattr(
+                        application.state,
+                        "fact_entity_association_repair_status",
+                        {},
+                    )
+                application.state.fact_entity_association_repair_status = {
+                    **durable,
+                    "state": "retrying",
+                    "source": "startup_background_repair",
+                    "batch_size": batch_size,
+                    "last_error": (
+                        durable.get("last_error") or type(exc).__name__
+                    ),
+                    "retry_attempt": consecutive_failures,
+                    "retry_delay_seconds": retry_delay,
+                }
+                await asyncio.sleep(retry_delay)
+                continue
+
+            consecutive_failures = 0
+            application.state.fact_entity_association_repair_status = {
+                **durable,
+                "source": "startup_background_repair",
+                "batch_size": batch_size,
+                "retry_attempt": 0,
+                "retry_delay_seconds": 0.0,
+            }
+            if durable["state"] == "complete":
+                return
+            await asyncio.sleep(max(0.0, float(tick_seconds)))
+    except asyncio.CancelledError:
+        raise
+    except Exception as exc:
+        durable = await asyncio.to_thread(get_repair_status, memory_db_path)
+        application.state.fact_entity_association_repair_status = {
+            **durable,
+            "source": "startup_background_repair",
+            "batch_size": batch_size,
+            "last_error": durable.get("last_error") or type(exc).__name__,
+        }
+
+
+async def _cancel_fact_entity_association_repair(application) -> None:
+    task = getattr(
+        application.state, "fact_entity_association_repair_task", None,
+    )
+    if task is None:
+        return
+    if not task.done():
+        task.cancel()
+    try:
+        await task
+    except asyncio.CancelledError:
+        pass
+
+
+def _schedule_source_quality_repair(
+    application,
+    memory_db_path: Path,
+    learning_db_path: Path,
+    *,
+    batch_size: int = 25,
+    tick_seconds: float = 1.0,
+) -> asyncio.Task:
+    """Schedule post-readiness repair without awaiting historical DB work."""
+    _set_source_quality_repair_status(
+        application,
+        state="scheduled",
+        source="startup_background_repair",
+        batch_size=batch_size,
+        profiles=[],
+        completed_profiles=[],
+        profile_results={},
+        batches_completed=0,
+        scanned=0,
+        observations=0,
+        last_error=None,
+    )
+    task = asyncio.create_task(
+        _source_quality_repair_loop(
+            application,
+            Path(memory_db_path),
+            Path(learning_db_path),
+            batch_size=batch_size,
+            tick_seconds=tick_seconds,
+        ),
+        name="source-quality-upgrade-repair",
+    )
+    application.state.source_quality_repair_task = task
+    return task
+
+
+async def _repair_one_source_quality_profile(
+    memory_db_path: Path,
+    learning_db_path: Path,
+    profile_id: str,
+    batch_size: int,
+) -> dict[str, int | bool]:
+    worker = asyncio.create_task(
+        asyncio.to_thread(
+            repair_historical_source_quality,
+            memory_db_path,
+            learning_db_path,
+            profile_id,
+            batch_size=batch_size,
+            max_batches=1,
+        ),
+        name=f"source-quality-repair-batch-{profile_id}",
+    )
+    try:
+        return await asyncio.shield(worker)
+    except asyncio.CancelledError:
+        # Cancellation cannot stop a running worker thread. Await the bounded
+        # batch so SQLite writes finish before daemon teardown closes storage.
+        await worker
+        raise
+
+
+def _record_source_quality_repair_result(
+    application,
+    profile_id: str,
+    result: dict[str, int | bool],
+) -> bool:
+    current = getattr(
+        application.state, "source_quality_repair_status", {},
+    )
+    completed = set(current.get("completed_profiles", []))
+    if result["complete"]:
+        completed.add(profile_id)
+    results = {
+        **current.get("profile_results", {}),
+        profile_id: result,
+    }
+    _set_source_quality_repair_status(
+        application,
+        profile_results=results,
+        completed_profiles=sorted(completed),
+        batches_completed=int(current.get("batches_completed", 0)) + 1,
+        scanned=int(current.get("scanned", 0)) + int(result["scanned"]),
+        observations=int(current.get("observations", 0))
+        + int(result["observations"]),
+    )
+    return not bool(result["complete"])
+
+
+async def _source_quality_repair_tick(
+    application,
+    memory_db_path: Path,
+    learning_db_path: Path,
+    profiles: list[str],
+    *,
+    batch_size: int,
+) -> list[str]:
+    _set_source_quality_repair_status(
+        application,
+        state="running",
+        profiles=profiles,
+        current_batch_size=batch_size,
+        last_error=None,
+    )
+    pending = []
+    for profile_id in profiles:
+        result = await _repair_one_source_quality_profile(
+            memory_db_path, learning_db_path, profile_id, batch_size,
+        )
+        if _record_source_quality_repair_result(
+            application, profile_id, result,
+        ):
+            pending.append(profile_id)
+        await asyncio.sleep(0)
+    return pending
+
+
+async def _discover_source_quality_profiles(
+    memory_db_path: Path,
+    pending: list[str] | None,
+    completed: list[str],
+) -> list[str]:
+    discovered = await asyncio.to_thread(
+        enumerate_source_quality_repair_profiles,
+        memory_db_path,
+    )
+    return sorted(
+        (set(pending or []) | set(discovered)) - set(completed),
+    )
+
+
+async def _source_quality_repair_loop(
+    application,
+    memory_db_path: Path,
+    learning_db_path: Path,
+    *,
+    batch_size: int,
+    tick_seconds: float,
+) -> None:
+    """Run one resumable repair batch per discovered profile and tick."""
+    pending: list[str] | None = None
+    next_refresh = 0.0
+    successful_ticks = 0
+    try:
+        while True:
+            try:
+                now = time.monotonic()
+                if pending is None or now >= next_refresh:
+                    status = getattr(
+                        application.state,
+                        "source_quality_repair_status",
+                        {},
+                    )
+                    pending = await _discover_source_quality_profiles(
+                        memory_db_path,
+                        pending,
+                        status.get("completed_profiles", []),
+                    )
+                    next_refresh = (
+                        now + _SOURCE_QUALITY_PROFILE_REFRESH_SECONDS
+                    )
+                adaptive_batch = min(
+                    _SOURCE_QUALITY_MAX_BATCH_SIZE,
+                    batch_size * (2 ** min(successful_ticks, 4)),
+                )
+                pending = await _source_quality_repair_tick(
+                    application,
+                    memory_db_path,
+                    learning_db_path,
+                    pending,
+                    batch_size=adaptive_batch,
+                )
+            except (SourceQualityRepairUnavailable, sqlite3.Error):
+                _set_source_quality_repair_status(
+                    application,
+                    state="retrying",
+                    last_error="storage_temporarily_unavailable",
+                )
+            except Exception as exc:  # F6 fix: broad catch keeps loop alive
+                # Any unexpected exception in a single tick (e.g. malformed JSON
+                # blob, AttributeError from a half-migrated schema) must not kill
+                # the entire repair loop.  Log and continue to next iteration.
+                logger.warning(
+                    "source-quality repair tick failed unexpectedly: %s — "
+                    "loop continues",
+                    exc,
+                    exc_info=True,
+                )
+                _set_source_quality_repair_status(
+                    application,
+                    state="retrying",
+                    last_error=type(exc).__name__,
+                )
+            else:
+                successful_ticks += 1
+            if pending == []:
+                _set_source_quality_repair_status(application, state="complete")
+                return
+            await asyncio.sleep(max(0.0, float(tick_seconds)))
+    except asyncio.CancelledError:
+        _set_source_quality_repair_status(application, state="cancelled")
+        raise
+    except Exception as exc:
+        logger.warning("source-quality startup repair failed: %s", exc)
+        _set_source_quality_repair_status(
+            application, state="failed", last_error=type(exc).__name__,
+        )
+
+
+async def _cancel_source_quality_repair(application) -> None:
+    task = getattr(application.state, "source_quality_repair_task", None)
+    if task is None:
+        return
+    if not task.done():
+        task.cancel()
+    try:
+        await task
+    except asyncio.CancelledError:
+        pass
+
+
 @asynccontextmanager
 async def lifespan(application: FastAPI):
     """Initialize engine, workers, and optional services on startup."""
@@ -1067,9 +1463,186 @@ async def lifespan(application: FastAPI):
             except Exception as exc:
                 logger.warning("Vector store backfill failed (non-fatal): %s", exc)
 
+        def _self_heal():
+            """v3.8.2 zero-pain self-heal.
+
+            On daemon start (especially right after a pip/npm upgrade of a
+            months-old database), silently restore full retrieval capability
+            with ZERO user action:
+              1. embed facts that were never embedded (NULL embedding column),
+              2. backfill key-expansion alt-keys (BM25 recall aid) via one
+                 bounded maintenance pass per profile,
+              3. index everything (including the just-embedded facts) into the
+                 sqlite-vec store.
+            Fully non-blocking (daemon thread) — recall keeps serving throughout.
+            Every step is bounded + idempotent, so on an already-complete DB the
+            whole pass is a fast no-op. Progress is exposed at /status.self_heal
+            so the dashboard can show a plain-language "Optimizing memory…" line.
+            """
+            import time as _t
+            global _SELF_HEAL_STATUS
+            _SELF_HEAL_STATUS = {
+                "state": "checking_components", "embeddings_backfilled": 0,
+                "expansion_backfilled": 0, "null_remaining": None,
+                "components": None,
+                "started_at": _t.time(), "finished_at": None,
+            }
+            # Step 0 (v3.8.2 "whole self-healer"): repair components that
+            # silently failed to install from the internet — BEFORE waiting on
+            # the embedder, since a missing embedding model would make that wait
+            # pointless. Downloads missing HF models (embedder/reranker, only
+            # when torch is present) and pip-installs sqlite-vec when the
+            # interpreter is user-writable. Bounded, fail-open, never sudo,
+            # never auto-pulls Ollama. Manual-only items are recorded for the
+            # dashboard "what's missing" report (GET /api/v3/components), not
+            # acted on here.
+            try:
+                from superlocalmemory.core import component_healer
+                heal_res = component_healer.heal_missing(
+                    config,
+                    on_progress=lambda k, m: logger.info(
+                        "Self-heal component[%s]: %s", k, m,
+                    ),
+                )
+                _SELF_HEAL_STATUS["components"] = heal_res
+                if heal_res["attempted"]:
+                    logger.info("Self-heal components: %s", heal_res)
+            except Exception as exc:
+                logger.warning(
+                    "Self-heal component check failed (non-fatal): %s", exc,
+                )
+            _SELF_HEAL_STATUS["state"] = "waiting_embedder"
+            for _ in range(120):  # up to ~60s for the embedder to warm
+                if _embedding_warm:
+                    break
+                _t.sleep(0.5)
+            try:
+                embedder = getattr(retrieval_eng, "_embedder", None) if retrieval_eng else None
+                db = engine._db
+                if embedder is None or db is None:
+                    _SELF_HEAL_STATUS["state"] = "skipped_no_embedder"
+                    return
+                # 1) Embed never-embedded facts (all profiles, the upgrade
+                #    headline). Looped: backfill is idempotent + bounded, and the
+                #    shared embedding worker can transiently return None under
+                #    startup contention (concurrent recall-warmup). Retrying until
+                #    the NULL count stops shrinking makes the heal converge
+                #    robustly rather than abandoning on one transient miss.
+                #    Facts that never embed (e.g. a document far over the model's
+                #    token limit) are left as-is after a bounded number of no-progress
+                #    attempts. No-op when there are no NULLs.
+                _SELF_HEAL_STATUS["state"] = "backfilling_embeddings"
+                try:
+                    from superlocalmemory.storage.embedding_migrator import (
+                        backfill_missing_embeddings,
+                    )
+                    # RECALL-PRIORITY THROTTLE: the embedding worker is a single
+                    # serialized subprocess shared with foreground recall. A
+                    # continuous backfill starves interactive query-embedding and
+                    # recalls time out. So: (a) tiny batches (short worker holds),
+                    # and (b) before each batch, defer while ANY user recall is in
+                    # flight — reusing the same recall_gate the pending materializer
+                    # uses. This keeps recall responsive throughout the heal (the
+                    # zero-pain requirement); the heal just takes a little longer.
+                    from superlocalmemory.core import recall_gate
+                    total_embedded = 0
+                    no_progress = 0
+                    for _attempt in range(500):
+                        # Absolute priority to user recalls: pause the heal while
+                        # a recall is active (bounded wait so we never wedge).
+                        _waited = 0.0
+                        while recall_gate.in_flight() > 0 and _waited < 30.0:
+                            _t.sleep(0.5)
+                            _waited += 0.5
+                        r = backfill_missing_embeddings(
+                            config, db, embedder, limit=5, all_profiles=True,
+                        )
+                        got = r.get("embedded", 0)
+                        total_embedded += got
+                        _SELF_HEAL_STATUS["embeddings_backfilled"] = total_embedded
+                        _SELF_HEAL_STATUS["null_remaining"] = r.get("remaining_null", 0)
+                        if r.get("remaining_null", 0) == 0:
+                            break
+                        if got == 0:
+                            no_progress += 1
+                            if no_progress >= 5:  # transient recovery exhausted
+                                break
+                            _t.sleep(3)  # let the worker settle, then retry
+                        else:
+                            no_progress = 0
+                            _t.sleep(0.5)  # brief pause between bursts
+                    if total_embedded:
+                        logger.info(
+                            "Self-heal: embedded %d previously-unembedded facts "
+                            "(%d remaining)", total_embedded,
+                            _SELF_HEAL_STATUS["null_remaining"],
+                        )
+                except Exception as exc:
+                    logger.warning("Self-heal embedding backfill failed (non-fatal): %s", exc)
+                # 2) Math/key-expansion maintenance is intentionally NOT run here:
+                #    run_maintenance triggers a full Langevin backfill over every
+                #    fact, which on a large legacy DB takes minutes and its CPU
+                #    burst inflates foreground recall latency during the heal
+                #    window. The startup heal stays lean (embeddings + vector
+                #    index — what makes facts findable again). Langevin/Sheaf/
+                #    key-expansion continue to converge on the periodic
+                #    MaintenanceScheduler exactly as before — unchanged behavior.
+                # 3) Index everything (incl. newly-embedded) into the vector store.
+                _SELF_HEAL_STATUS["state"] = "indexing_vectors"
+                try:
+                    _backfill_vector_store()
+                except Exception as exc:
+                    logger.warning("Self-heal vector index failed (non-fatal): %s", exc)
+                _SELF_HEAL_STATUS["state"] = "complete"
+                _SELF_HEAL_STATUS["finished_at"] = _t.time()
+                logger.info("Self-heal complete: %s", _SELF_HEAL_STATUS)
+            except Exception as exc:
+                _SELF_HEAL_STATUS["state"] = "error"
+                logger.warning("Self-heal failed (non-fatal): %s", exc)
+
+        def _component_recheck_loop():
+            """Periodic component re-check (v3.8.2 "whole self-healer").
+
+            The startup heal (_self_heal Step 0) runs once. This keeps the
+            self-healer promise for a long-running daemon: it re-probes
+            components on a slow cadence and auto-repairs any that regress
+            (a cache eviction, a half-finished install). No-op on a healthy
+            machine — nothing is auto-fixable-missing, so it is just a cheap
+            probe. Disable with SLM_COMPONENT_RECHECK_SEC=0.
+            """
+            import os as _os
+            import time as _t
+            try:
+                cadence = int(_os.environ.get("SLM_COMPONENT_RECHECK_SEC", "1800"))
+            except ValueError:
+                cadence = 1800
+            if cadence <= 0:
+                return
+            # Let the startup heal + warmup settle before the first re-check.
+            _t.sleep(max(cadence, 300))
+            while True:
+                try:
+                    from superlocalmemory.core import component_healer
+                    res = component_healer.heal_missing(
+                        config,
+                        on_progress=lambda k, m: logger.info(
+                            "Component re-check[%s]: %s", k, m,
+                        ),
+                    )
+                    if res["attempted"]:
+                        logger.info("Component re-check repaired: %s", res)
+                except Exception as exc:
+                    logger.debug("Component re-check failed (non-fatal): %s", exc)
+                _t.sleep(cadence)
+
         threading.Thread(target=_warmup_embedder, daemon=True, name="embed-warmup").start()
         threading.Thread(target=_warmup_recall, daemon=True, name="recall-warmup").start()
-        threading.Thread(target=_backfill_vector_store, daemon=True, name="vs-backfill").start()
+        # v3.8.2: self-heal supersedes the bare vector-store backfill (it calls
+        # _backfill_vector_store itself, after embedding + expansion heal).
+        threading.Thread(target=_self_heal, daemon=True, name="self-heal").start()
+        threading.Thread(
+            target=_component_recheck_loop, daemon=True, name="component-recheck",
+        ).start()
 
         # v3.6.8: Runtime recall-health monitor. The three warmups above run
         # ONCE at boot; on a long-running daemon the graph page cache gets
@@ -1350,7 +1923,20 @@ async def lifespan(application: FastAPI):
         application.state.daemon_descriptor = _publish_process_descriptor(
             _configured_daemon_port(), SLM_VERSION, "ready",
         )
-        yield
+        _schedule_source_quality_repair(
+            application,
+            state_path("memory.db"),
+            state_path("learning.db"),
+        )
+        _schedule_fact_entity_association_repair(
+            application,
+            state_path("memory.db"),
+        )
+        try:
+            yield
+        finally:
+            await _cancel_fact_entity_association_repair(application)
+            await _cancel_source_quality_repair(application)
 
     # Cancel the cross-platform sync loop (H-CONC-2) so adapter file I/O does
     # not outlive the daemon.
@@ -1945,19 +2531,35 @@ def _register_dashboard_routes(application: FastAPI) -> None:
                         "error": "Remote HTTP MCP requires a configured SLM API key."
                     },
                 )
-            # v3.6.12 (csrf-1): defense-in-depth CSRF/DNS-rebinding guard on
-            # state-changing requests. A cross-origin browser Origin is rejected;
-            # loopback origins (the local dashboard) always pass, and LAN origins
-            # pass only when explicitly allowlisted in SLM_REMOTE mode. Non-browser
-            # clients (CLI/MCP/curl) send no Origin and are unaffected.
+            # Defense-in-depth CSRF/DNS-rebinding guard. A loopback hostname is
+            # not, by itself, a trusted web origin: a different local process can
+            # serve a page on another port. Credentialless browser writes must
+            # therefore originate from this daemon's exact port. A local
+            # integration on another port may still write when it presents a
+            # valid credential; require_http_mutation_actor below validates it.
+            # LAN origins remain opt-in through remote mode. Non-browser clients
+            # (CLI/MCP/curl) send no Origin and are unaffected.
             if requires_mutation_actor:
                 _origin = headers.get("origin", "") or headers.get("Origin", "")
                 if _origin:
-                    _ok_origin = any(_origin.startswith(p) for p in (
-                        "http://127.0.0.1", "https://127.0.0.1",
-                        "http://localhost", "https://localhost",
-                        "http://[::1]", "https://[::1]",
-                    ))
+                    from superlocalmemory.server.origin import (
+                        origin_is_daemon,
+                        origin_is_loopback,
+                    )
+
+                    _daemon = getattr(application.state, "daemon_descriptor", None)
+                    _daemon_port = getattr(_daemon, "port", None) or _configured_daemon_port()
+                    _ok_origin = origin_is_daemon(_origin, port=int(_daemon_port))
+                    _has_browser_credential = any(
+                        headers.get(_header)
+                        for _header in (
+                            "x-slm-daemon-capability",
+                            "x-install-token",
+                            "x-slm-api-key",
+                        )
+                    )
+                    if not _ok_origin and origin_is_loopback(_origin) and _has_browser_credential:
+                        _ok_origin = True
                     if not _ok_origin:
                         from superlocalmemory.core.remote_mode import is_remote_origin_allowed
                         _ok_origin = is_remote_origin_allowed(_origin)
@@ -2038,30 +2640,11 @@ def _register_dashboard_routes(application: FastAPI) -> None:
             # single-operator installs are unaffected. Owner (no session) reads
             # freely unless company mode (require_login) is on; a logged-in user
             # must hold READ on the active workspace.
-            _p = request.url.path
-            _is_sensitive_read = (
-                (request.method == "GET" and _p.startswith((
-                    "/api/memories", "/api/facts", "/api/clusters", "/api/graph",
-                    "/api/v3/associations", "/api/v3/core-memory",
-                    "/api/v3/soft-prompts",
-                    # Config metadata GETs expose the install path (base_dir) and
-                    # LLM stack (provider/model/endpoint). Harmless to the loopback
-                    # owner (personal mode: gate is a no-op), but in company mode an
-                    # unauthenticated caller must not read them — same login gate as
-                    # content reads.
-                    "/api/v3/dashboard", "/api/v3/mode", "/api/v3/embedding/config",
-                    # SEC-M-02: the remaining config GETs also expose base_dir,
-                    # daemon port, and backend topology — gate them in company mode.
-                    "/api/v3/scope/config", "/api/v3/storage/config",
-                    "/api/v3/daemon/config", "/api/v3/mesh/config",
-                    "/api/v3/trust/config", "/api/v3/forgetting/config",
-                    # MCP profile metadata — reveals which tools agent clients
-                    # can invoke; consistent with other config GETs.
-                    "/api/v3/mcp/profiles")))
-                or _p in ("/api/search", "/api/v3/recall/trace")
-                or _p.startswith("/api/v3/recall")
-            )
-            if _is_sensitive_read:
+            # Config, learning, and behavioral reads expose installation,
+            # preference, workflow, source-reputation, or outcome data.
+            if _is_sensitive_dashboard_read(
+                request.method, request.url.path,
+            ):
                 _resp = _rbac_read_gate(request, application.state)
                 if _resp is not None:
                     return _resp
@@ -2370,7 +2953,15 @@ def _register_daemon_routes(application: FastAPI) -> None:
         request: Request,
         q: str = "", query: str = "", limit: int = CANONICAL_RECALL_LIMIT,
         session_id: str = "",
-        fast: bool = False,
+        # v3.8.2 client-driven agentic: ``fast`` is left UNSET (None) by default
+        # so the daemon resolves the configured policy (retrieval.client_driven_agentic,
+        # ships True). The agent hot path is consumed by a frontier LLM that
+        # reformulates queries far better than the local Ollama model, so it
+        # skips the internal agentic round and returns fast local retrieval (all
+        # six channels + reranker) plus confidence signals; the client re-queries
+        # on low confidence. An explicit ?fast=true / ?fast=false always wins.
+        # See recall_pipeline.resolve_hot_path_fast.
+        fast: bool | None = None,
         full: bool = False,
         include_source: bool = False,
         include_global: bool | None = None,
@@ -2382,6 +2973,10 @@ def _register_daemon_routes(application: FastAPI) -> None:
         engine = _get_engine_or_503()
         if not search_query:
             return {"results": [], "count": 0, "query_type": "none", "retrieval_time_ms": 0}
+        # v3.8.2: resolve the client-driven-agentic default now so the concrete
+        # bool drives BOTH the full-recall semaphore below and engine.recall().
+        from superlocalmemory.core.recall_pipeline import resolve_hot_path_fast
+        fast = resolve_hot_path_fast(fast, engine._config)
         # S9-DASH-02: session_id for the outcome-queue producer.
         # Priority: ?session_id= > X-SLM-Session-Id header > synthetic
         # "http:<ts>". Without a session_id the recall still works
@@ -2409,7 +3004,7 @@ def _register_daemon_routes(application: FastAPI) -> None:
         # recall (reranker timeout, cold embedder) blocks ALL endpoints.
         import asyncio
         _begin_recall()
-        # v3.4.53: Full (non-fast) recalls are gated by a semaphore to
+        # v3.4.53: Opt-in deep recalls are gated by a semaphore to
         # prevent resource oversaturation. Ollama serialises concurrent
         # embedding calls and the reranker subprocess has a single lock —
         # queuing more than ~3 concurrent full recalls just adds latency.
@@ -2526,7 +3121,7 @@ def _register_daemon_routes(application: FastAPI) -> None:
             if isinstance(extra, dict):
                 meta.update(extra)
             command = build_engine_ingestion_command(engine)
-            receipt = command.submit(IngestionRequest(
+            ingestion_request = IngestionRequest(
                 content=req.content,
                 profile_id=engine._profile_id,
                 source_type="http",
@@ -2536,9 +3131,42 @@ def _register_daemon_routes(application: FastAPI) -> None:
                 shared_with=tuple(shared_with or ()),
                 trusted_actor_id=trusted_actor_id,
                 session_id=req.session_id,
-            ))
+            )
+            # SQLite admission is usually milliseconds, but it can wait on a
+            # concurrent migration or writer.  Keep that wait out of ASGI so
+            # dashboard navigation and recall stay responsive.
+            receipt = await asyncio.to_thread(command.submit, ingestion_request)
+            result = receipt
+            wait_budget_exhausted = False
+            if wait:
+                materialization_task = asyncio.create_task(
+                    asyncio.to_thread(command.materialize, receipt.operation_id)
+                )
+                try:
+                    result = await asyncio.wait_for(
+                        asyncio.shield(materialization_task),
+                        timeout=_REMEMBER_ENRICHMENT_WAIT_SECONDS,
+                    )
+                except TimeoutError:
+                    # The task retains the M018 lease and continues outside
+                    # this request.  Return the durable receipt honestly;
+                    # the normal materializer can also reclaim it after a
+                    # lease expiry if the request-owned worker dies.
+                    wait_budget_exhausted = True
+
+                    def _log_background_materialization(task):
+                        try:
+                            task.result()
+                        except Exception as exc:
+                            logger.warning(
+                                "bounded remember enrichment failed for %s: %s",
+                                receipt.operation_id,
+                                exc,
+                            )
 
-            result = command.materialize(receipt.operation_id) if wait else receipt
+                    materialization_task.add_done_callback(
+                        _log_background_materialization
+                    )
             fact_ids = list(result.fact_ids)
             # The queryable write is a separate durable transaction.  A cold
             # optional enrichment dependency (most often the local embedding
@@ -2581,10 +3209,13 @@ def _register_daemon_routes(application: FastAPI) -> None:
                 "note": (
                     "canonical ingestion complete"
                     if completed
+                    else "queryable now; enrichment continues after the wait budget"
+                    if wait_budget_exhausted
                     else "queryable now; canonical enrichment will retry"
                     if enrichment_deferred
                     else "queryable now; canonical enrichment pending"
                 ),
+                "wait_budget_exhausted": wait_budget_exhausted,
             }
         except Exception as exc:
             raise HTTPException(500, detail=str(exc))
@@ -2751,8 +3382,68 @@ def _register_daemon_routes(application: FastAPI) -> None:
             "legacy_port": _LEGACY_PORT,
             "profile": profile_snapshot.profile_id,
             "profile_generation": profile_snapshot.generation,
+            # F2 fix: expose M028 backfill progress so operators can monitor
+            # the post-upgrade fact/entity association repair state.
+            "m028_backfill": getattr(
+                application.state,
+                "fact_entity_association_repair_status",
+                None,
+            ),
+            # v3.8.2: zero-pain self-heal progress (embeddings/expansion/vector
+            # index backfill after an upgrade). Dashboard renders a plain
+            # "Optimizing memory…" line from this. Defaults to idle before start.
+            "self_heal": globals().get("_SELF_HEAL_STATUS", {"state": "idle"}),
         }
 
+    @application.get("/api/v3/components")
+    async def components():
+        """Component / dependency health (v3.8.2).
+
+        Read-only snapshot from the central registry (core.component_registry)
+        — the same source the self-heal thread acts on. Powers the dashboard
+        'what's missing' report and `slm doctor`. Includes live 'retrying'
+        overlays while a background repair is in flight. Never mutates state.
+        """
+        _update_activity()
+        try:
+            from superlocalmemory.core import component_registry
+            cfg = getattr(application.state, "config", None)
+            return component_registry.snapshot(cfg)
+        except Exception as exc:
+            raise HTTPException(500, detail=str(exc))
+
+    @application.post("/api/v3/components/heal")
+    async def heal_components(request: Request):
+        """Trigger a component self-heal pass (dashboard 'Retry now').
+
+        Repairs auto-fixable missing components (re-download missing models,
+        install sqlite-vec). Runs in a background thread and returns
+        immediately so the HTTP request never blocks on a multi-minute
+        download; the dashboard polls GET /api/v3/components to watch the
+        'retrying' → 'ok' transition. Safe to call repeatedly (no-op when
+        healthy). Requires the dashboard write principal, like other
+        dashboard mutations.
+        """
+        _require_write_actor(request)
+        _update_activity()
+        cfg = getattr(application.state, "config", None)
+
+        def _run():
+            try:
+                from superlocalmemory.core import component_healer
+                res = component_healer.heal_missing(
+                    cfg,
+                    on_progress=lambda k, m: logger.info(
+                        "Manual heal[%s]: %s", k, m,
+                    ),
+                )
+                logger.info("Manual component heal: %s", res)
+            except Exception as exc:
+                logger.warning("Manual component heal failed (non-fatal): %s", exc)
+
+        threading.Thread(target=_run, daemon=True, name="manual-heal").start()
+        return {"status": "started"}
+
     @application.get("/list")
     async def list_facts(limit: int = 50):
         _update_activity()