superlocalmemory 3.8.2 → 3.8.5

package/src/superlocalmemory/mcp/http_transport.py

@@ -2,22 +2,261 @@
 # Licensed under AGPL-3.0-or-later - see LICENSE file
 # Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
 
-"""Resource-safe FastMCP Streamable-HTTP integration.
+"""Resource-safe FastMCP Streamable-HTTP integration — Workstream E additions.
 
 MCP SDK 1.27.1 passes an AnyIO ``MemoryObjectReceiveStream`` directly to
 ``EventSourceResponse`` for every JSON-RPC POST.  The response consumes that
 stream but does not close it after normal iteration, leaving one receive
 endpoint per request for the garbage collector.  The response is the owner of
 that per-request iterator, so SLM closes it at the response boundary.
+
+Workstream E (3.8.4): MCP Connection Resilience
+------------------------------------------------
+Two root causes of frequent MCP disconnects are addressed here:
+
+RC-2: No session idle timeout — zombie sessions accumulate forever.
+    Fix: ``SLMFastMCP.streamable_http_app()`` pre-creates the
+    ``StreamableHTTPSessionManager`` with a finite ``session_idle_timeout``
+    (default 600 s, overridable via ``SLM_MCP_SESSION_IDLE_TIMEOUT_S``).
+
+RC-3: No EventStore — every SSE drop requires a full re-initialize.
+    Fix: ``SLMInMemoryEventStore`` (bounded, in-memory, async-safe) is
+    injected as the session manager's event store.  A dropped SSE stream can
+    resume via ``Last-Event-ID`` instead of forcing a new ``initialize``
+    handshake.
+
+KNOWN LIMITATIONS (not fixed here, documented for future work):
+    * RC-1 (mcp-remote orphan test-sessions): The mcp-remote v0.1.38 bug
+      creates a ``testTransport``/``testClient`` that is never closed.  This
+      leaks one zombie session per mcp-remote startup.  Tracked upstream.
+      Session idle-timeout mitigates the accumulation.
+    * Client reconnect after daemon restart: ``SLMInMemoryEventStore`` is
+      in-memory only and does not survive daemon process restart.  After a
+      restart, stale ``Last-Event-ID`` values are unknown to the new store;
+      ``replay_events_after`` returns ``None`` and the client must
+      re-initialize.  A SQLite-backed EventStore is planned for v3.9.
+    * Claude Code client bug Anthropic #48557: Claude Code may send a stale
+      MCP session ID after server restart.  This is a client-side defect;
+      SLM cannot fix it from the server.
+    * Event-loop stall during background maintenance (RC-6): The pruner-lock
+      stall fix lives in Workstream A+F (already merged into this branch).
+      This module depends on that fix; see fix/3.8.4 merge commit.
 """
 
 from __future__ import annotations
 
+import logging
+import os
+from collections import OrderedDict, deque
+from typing import Any
+
 from mcp.server.fastmcp import FastMCP
+from mcp.server.streamable_http import (
+    EventCallback,
+    EventId,
+    EventMessage,
+    EventStore,
+    StreamId,
+)
 from sse_starlette.sse import EventSourceResponse
 from starlette.types import Receive, Scope, Send
+
 from superlocalmemory import __version__
 
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# Session idle timeout — configurable default
+# ---------------------------------------------------------------------------
+
+#: Default session idle timeout in seconds.
+#:
+#: Rationale for 600 s (10 minutes), not the SDK's suggested 1800 s:
+#:   * At 1800 s, with mcp-remote leaking 3 zombie sessions per conversation
+#:     start and a typical conversation cadence of one every 15 min,
+#:     up to (1800/15) × 3 = 360 zombie sessions can accumulate in the worst
+#:     case before the first one expires.  600 s bounds this to ~6.
+#:   * 600 s is well above the 15 s SSE keepalive interval, the typical
+#:     in-conversation idle gap (<5 min), and the mcp-remote reconnect window.
+#:   * Active sessions push their idle deadline forward on every tool call,
+#:     so a user making any request within 10 min never loses their session.
+#:   * A session idle for exactly 10 min (user stepped away) is reaped and
+#:     recreated transparently on next tool call via mcp-remote reconnect.
+#:
+#: CRIT — anti-patterns to avoid:
+#:   Too aggressive (<60 s): frequent idle-deadline checks waste CPU in
+#:     AnyIO's CancelScope machinery; legitimate quiet users are interrupted.
+#:   Too lenient (>7200 s = 2 h): zombie sessions accumulate at rates that
+#:     can reach 100+ entries, degrading AnyIO task-group scheduling.
+_DEFAULT_SESSION_IDLE_TIMEOUT: float = 600.0
+
+
+def _slm_session_idle_timeout() -> float:
+    """Return the configured MCP session idle timeout in seconds.
+
+    Reads ``SLM_MCP_SESSION_IDLE_TIMEOUT_S`` from the environment.
+    Falls back to :data:`_DEFAULT_SESSION_IDLE_TIMEOUT` on missing or
+    non-numeric values.  A value ≤ 0 is also treated as the default.
+    """
+    raw = os.environ.get("SLM_MCP_SESSION_IDLE_TIMEOUT_S", "").strip()
+    if raw:
+        try:
+            val = float(raw)
+            if val > 0:
+                return val
+        except ValueError:
+            logger.warning(
+                "SLM_MCP_SESSION_IDLE_TIMEOUT_S=%r is not a valid number; "
+                "using default %s s",
+                raw,
+                _DEFAULT_SESSION_IDLE_TIMEOUT,
+            )
+    return _DEFAULT_SESSION_IDLE_TIMEOUT
+
+
+# ---------------------------------------------------------------------------
+# SLMInMemoryEventStore — bounded, in-memory, async-safe
+# ---------------------------------------------------------------------------
+
+
+class SLMInMemoryEventStore(EventStore):
+    """Bounded in-memory event store for MCP SSE resumability.
+
+    Enables clients to reconnect to the GET /mcp SSE endpoint and replay
+    missed events via the ``Last-Event-ID`` header instead of performing a
+    full ``initialize`` handshake (which creates a new session and adds to
+    the zombie-session count).
+
+    Design choices:
+    ---------------
+    * **Per-stream bounded deque** — each stream_id maps to a
+      ``deque(maxlen=max_events_per_stream)``.  Oldest events are evicted
+      automatically (FIFO) when the cap is hit.
+    * **Global stream count cap** — the store tracks at most
+      ``max_streams`` distinct stream_ids.  When exceeded, the least-recently
+      used stream is evicted from the dict.  This bounds total memory usage.
+    * **Async-safe, lock-free** — all access happens on a single asyncio event
+      loop; no ``asyncio.Lock`` is needed.
+    * **Priming events (message=None) are stored but skipped during replay** —
+      the SDK mints a fresh priming event in the replay path (see
+      ``StreamableHTTPServerTransport._replay_events``).
+
+    Memory bound:
+    -------------
+    ``max_events_per_stream`` × ~2 KB (avg JSONRPCMessage) × ``max_streams``
+    = 200 × 2 KB × 100 = ~40 MB in the absolute worst case.
+    At steady state with idle-timeout reaping sessions after 10 min, the
+    typical load is 3–6 active streams × 200 events × 2 KB ≈ 1.2–2.4 MB.
+
+    Known limitation:
+    -----------------
+    This store is in-memory only.  It does not survive a daemon process
+    restart.  After a restart, any ``Last-Event-ID`` from a previous process
+    is unknown; ``replay_events_after`` returns ``None`` and the client must
+    re-initialize.  A SQLite-backed EventStore is planned for v3.9.
+    """
+
+    def __init__(
+        self,
+        max_events_per_stream: int = 200,
+        max_streams: int = 100,
+    ) -> None:
+        """Initialise the bounded event store.
+
+        Args:
+            max_events_per_stream: Maximum number of events stored per
+                stream before oldest events are dropped.  Default 200
+                (~400 KB per stream at 2 KB/event).
+            max_streams: Maximum number of distinct stream_ids tracked.
+                When exceeded, the least-recently used stream is evicted
+                (all its events are lost).  Default 100.
+        """
+        self._max_events = max_events_per_stream
+        self._max_streams = max_streams
+        # OrderedDict maintains insertion/access order for LRU eviction.
+        # stream_id → deque[(event_id, message)]
+        self._store: OrderedDict[str, deque] = OrderedDict()
+        # Monotonically incrementing counter; single event loop → no lock needed.
+        self._counter: int = 0
+
+    # ------------------------------------------------------------------
+    # EventStore ABC implementation
+    # ------------------------------------------------------------------
+
+    async def store_event(
+        self,
+        stream_id: StreamId,
+        message: Any,  # JSONRPCMessage | None
+    ) -> EventId:
+        """Store an event for the given stream and return its unique event_id.
+
+        Args:
+            stream_id: The stream (GET /mcp SSE connection) this event belongs to.
+            message: The JSON-RPC message, or ``None`` for priming events.
+
+        Returns:
+            A monotonically incrementing string event_id.
+        """
+        self._counter += 1
+        event_id: EventId = str(self._counter)
+
+        if stream_id not in self._store:
+            # Evict oldest stream if at capacity
+            if len(self._store) >= self._max_streams:
+                oldest_stream, _ = self._store.popitem(last=False)
+                logger.debug(
+                    "SLMInMemoryEventStore: evicted oldest stream %r "
+                    "(max_streams=%d reached)",
+                    oldest_stream,
+                    self._max_streams,
+                )
+            self._store[stream_id] = deque(maxlen=self._max_events)
+        else:
+            # Move to "most recently used" end so LRU eviction works correctly
+            self._store.move_to_end(stream_id)
+
+        self._store[stream_id].append((event_id, message))
+        return event_id
+
+    async def replay_events_after(
+        self,
+        last_event_id: EventId,
+        send_callback: EventCallback,
+    ) -> StreamId | None:
+        """Replay events that occurred after ``last_event_id``.
+
+        Searches all tracked streams for ``last_event_id`` and forwards every
+        subsequent non-None event to ``send_callback``.  Priming events
+        (``message=None``) are skipped; the SDK mints a fresh priming event
+        in the replay path.
+
+        Args:
+            last_event_id: The ID of the last event the client received.
+            send_callback: Async callback that receives each missed
+                ``EventMessage`` in order.
+
+        Returns:
+            The stream_id that contained ``last_event_id``, or ``None`` if
+            the event was not found (client must re-initialize).
+        """
+        for stream_id, events in self._store.items():
+            found = False
+            for event_id, message in events:
+                if found:
+                    if message is not None:
+                        await send_callback(EventMessage(message=message, event_id=event_id))
+                elif event_id == last_event_id:
+                    found = True
+            if found:
+                return stream_id
+        # last_event_id not in any stream (evicted or never seen)
+        return None
+
+
+# ---------------------------------------------------------------------------
+# SSE resource guard (unchanged from pre-E)
+# ---------------------------------------------------------------------------
+
 
 class ClosingEventSourceResponse(EventSourceResponse):
     """EventSourceResponse that closes the async iterator it consumes."""
@@ -38,8 +277,35 @@ def install_streamable_http_resource_guard() -> None:
     streamable_http.EventSourceResponse = ClosingEventSourceResponse
 
 
+# ---------------------------------------------------------------------------
+# SLMFastMCP — session lifecycle + SSE cleanup
+# ---------------------------------------------------------------------------
+
+
 class SLMFastMCP(FastMCP):
-    """FastMCP with SLM release identity and deterministic SSE cleanup."""
+    """FastMCP with SLM release identity, deterministic SSE cleanup, and
+    session lifecycle management (Workstream E).
+
+    Additions over the base FastMCP:
+
+    1. **Session idle timeout**: the ``StreamableHTTPSessionManager`` is
+       pre-created with a finite ``session_idle_timeout`` so zombie sessions
+       (from mcp-remote orphan test-connects, or abandoned conversations) are
+       automatically reaped instead of accumulating forever.
+
+    2. **Bounded event store**: a ``SLMInMemoryEventStore`` is injected so
+       that a dropped SSE stream can resume via ``Last-Event-ID`` without a
+       full ``initialize`` round-trip.
+
+    3. **Ordering guard**: ``streamable_http_app()`` reads the ``stateless_http``
+       flag from ``self.settings`` at call time.  If the flag is ``True``
+       (set by ``_configure_mcp_transport_settings()`` in unified_daemon.py),
+       ``session_idle_timeout`` is suppressed — the MCP SDK raises
+       ``RuntimeError`` if both are set simultaneously.  If
+       ``streamable_http_app()`` is called before settings are configured
+       (e.g. in unit tests), the default ``stateless_http=False`` applies
+       safely.
+    """
 
     def __init__(self, *args, product_version: str = __version__, **kwargs) -> None:
         super().__init__(*args, **kwargs)
@@ -49,6 +315,72 @@ class SLMFastMCP(FastMCP):
         # (for example) 1.27.1 to every IDE client.
         self._mcp_server.version = product_version
 
-    def streamable_http_app(self):
+    def streamable_http_app(self):  # type: ignore[override]
+        """Return the Streamable-HTTP Starlette app.
+
+        Pre-creates the ``StreamableHTTPSessionManager`` with SLM-specific
+        parameters before delegating to ``super()`` (which skips re-creation
+        because the manager is already set).
+
+        Ordering guarantee
+        ------------------
+        In production ``unified_daemon.py`` always calls
+        ``_configure_mcp_transport_settings(fastmcp)`` *before* this method,
+        so ``self.settings.stateless_http`` reflects the correct runtime value.
+        If this method is called first (unit tests, embedded hosts), the
+        default ``stateless_http=False`` is used — safe because the stateless
+        guard only matters when ``stateless_http=True`` (avoid SDK RuntimeError).
+
+        Dependency note (A+F)
+        ---------------------
+        This method assumes that ``fix/3.8.4-A+F`` (already merged into this
+        branch) has eliminated the pruner-lock stall that caused event-loop
+        starvation during background maintenance.  Session idle-timeout and
+        the event store improve resilience to transient drops, but they cannot
+        compensate for a fully stalled event loop.
+        """
         install_streamable_http_resource_guard()
+
+        if self._session_manager is None:
+            from mcp.server.streamable_http_manager import StreamableHTTPSessionManager
+
+            # Ordering guard: read stateless_http defensively.
+            # ``_configure_mcp_transport_settings()`` may not have been called
+            # yet; ``getattr`` with a False default ensures we never pass
+            # session_idle_timeout=<value> into a stateless manager (SDK raises
+            # RuntimeError if both are set simultaneously).
+            is_stateless: bool = getattr(self.settings, "stateless_http", False)
+
+            # Idle timeout: finite for stateful sessions; None for stateless
+            # (stateless sessions have no identity — there is nothing to reap).
+            idle_timeout: float | None = (
+                None if is_stateless else _slm_session_idle_timeout()
+            )
+
+            # Event store: inject SLMInMemoryEventStore for stateful mode.
+            # Stateless mode must not receive an event store (SDK limitation).
+            # If the caller already configured an event store (via FastMCP
+            # constructor argument), respect it — do not replace with ours.
+            if is_stateless:
+                event_store: EventStore | None = None
+            else:
+                event_store = self._event_store or SLMInMemoryEventStore()
+
+            self._session_manager = StreamableHTTPSessionManager(
+                app=self._mcp_server,
+                event_store=event_store,
+                retry_interval=self._retry_interval,
+                json_response=self.settings.json_response,
+                stateless=is_stateless,
+                security_settings=self.settings.transport_security,
+                session_idle_timeout=idle_timeout,
+            )
+            logger.info(
+                "SLM MCP session manager created: stateless=%s, "
+                "idle_timeout=%ss, event_store=%s",
+                is_stateless,
+                idle_timeout,
+                type(event_store).__name__ if event_store else "None",
+            )
+
         return super().streamable_http_app()

package/src/superlocalmemory/retrieval/engine.py

@@ -955,7 +955,13 @@ class RetrievalEngine:
         Blended: alpha * sigmoid(CE_score) + (1 - alpha) * rrf_score.
         Speaker tags stripped before scoring (Bug 3 fix).
         """
-        # Bug 2 fix: score ALL candidates, not just top_k
+        # Bug 2 fix: score ALL candidates, not just top_k. v3.8.5: verified on
+        # the real DB that bounding the CE to the top-N fusion candidates both
+        # (a) gave NO latency win (the cross-encoder batches all pairs in one
+        # forward pass, so 60 vs 184 pairs is within noise) and (b) CHANGED the
+        # top-5 on 4/8 queries — the CE legitimately promotes items ranked below
+        # the fusion top-N into the answer. So exhaustive reranking stays: it is
+        # a quality feature, not the latency bottleneck.
         candidates = [
             (fact_map[fr.fact_id], fr.fused_score)
             for fr in fused if fr.fact_id in fact_map

package/src/superlocalmemory/retrieval/entity_channel.py

@@ -14,6 +14,7 @@ from __future__ import annotations
 
 import json
 import logging
+import os
 import re
 import threading
 from collections import defaultdict
@@ -31,6 +32,24 @@ if TYPE_CHECKING:
 
 logger = logging.getLogger(__name__)
 
+
+def _adj_ttl_seconds() -> float:
+    """In-memory adjacency-cache TTL (seconds), env-overridable.
+
+    v3.8.5: raised from a hard-coded 300s to 3600s. The TTL only exists to
+    catch edge-WEIGHT mutations (pruning / MAX-merge) that leave the edge COUNT
+    unchanged — new memories already force a reload via the count check. At 300s
+    a 208K-edge graph rebuilt on the recall hot path every 5 idle minutes,
+    causing a recurring multi-second latency spike. Weight drift is a minor
+    ranking refinement, so a longer TTL trades negligible staleness for a big
+    latency win. Set SLM_ENTITY_ADJ_TTL_S to tune (0 disables time-based reload;
+    the count-based correctness reload always remains).
+    """
+    try:
+        return max(0.0, float(os.environ.get("SLM_ENTITY_ADJ_TTL_S", "3600")))
+    except (TypeError, ValueError):
+        return 3600.0
+
 _PROPER_NOUN_RE = re.compile(r"\b[A-Z][a-z]{1,}\b")
 
 _ENTITY_STOP: frozenset[str] = frozenset({
@@ -156,7 +175,12 @@ class EntityGraphChannel:
         # count-stable window would otherwise serve a stale adjacency map.
         import time as _t_ec
         _now_ec = _t_ec.monotonic()
-        _fresh = (_now_ec - getattr(self, "_adj_loaded_at", 0.0)) < 300.0
+        _ttl = _adj_ttl_seconds()
+        # TTL=0 disables the time-based reload entirely (count-based correctness
+        # reload still applies); otherwise the cache is fresh within the TTL.
+        _fresh = _ttl <= 0.0 or (
+            (_now_ec - getattr(self, "_adj_loaded_at", 0.0)) < _ttl
+        )
         if (self._adj_scope_key == scope_key
                 and (self._adj or self._visible_fact_ids)
                 and self._adj_edge_count == current_count

package/src/superlocalmemory/retrieval/reranker.py

@@ -68,6 +68,21 @@ _SUBPROCESS_RESPONSE_TIMEOUT = 15  # v3.4.52: 15s (was 180s). Long timeout block
 # scores without reranking.
 _WORKER_RECYCLE_AFTER = 500  # Recycle after N requests
 
+# One-time model load is far heavier than a live rerank request: the child
+# process imports torch / sentence-transformers and runs a warmup inference,
+# which measured 9-16s on the reference machine.  Sharing the 15s live-request
+# timeout (``_SUBPROCESS_RESPONSE_TIMEOUT``) made the load a coin flip — logs
+# showed ~half of daemon boots hitting "timed out after 15s", killing the
+# worker, and leaving recall on FALLBACK scoring for the entire daemon
+# lifetime (a silent quality regression, not a transient one).  The load gets
+# its own generous budget, and the background warmup RETRIES with backoff so a
+# transient slow/failed load self-heals instead of permanently degrading
+# recall quality.  Live rerank requests keep the tight 15s cap so a recall
+# never blocks on a sick subprocess.
+_WARMUP_LOAD_TIMEOUT = int(os.environ.get("SLM_RERANKER_WARMUP_TIMEOUT", "90"))
+_WARMUP_MAX_ATTEMPTS = int(os.environ.get("SLM_RERANKER_WARMUP_ATTEMPTS", "5"))
+_WARMUP_RETRY_BACKOFF_S = float(os.environ.get("SLM_RERANKER_WARMUP_BACKOFF", "3"))
+
 
 class CrossEncoderReranker:
     """Rerank candidate facts using a local cross-encoder model.
@@ -133,20 +148,72 @@ class CrossEncoderReranker:
 
         def _warmup() -> None:
             try:
-                self._ensure_worker()
-                if self._worker_proc is None:
-                    return
-                resp = self._send_request({
-                    "cmd": "load",
-                    "model_name": self._model_name,
-                    "backend": self._backend,
-                }, timeout=_SUBPROCESS_RESPONSE_TIMEOUT)
-                if resp and resp.get("ok"):
-                    self._model_loaded = True
-                    logger.info(
-                        "Reranker worker warm (backend=%s, warmup_inference=%s)",
-                        resp.get("backend", "?"),
-                        resp.get("warmup_inference", False),
+                for attempt in range(1, _WARMUP_MAX_ATTEMPTS + 1):
+                    if self._model_loaded:
+                        return
+                    try:
+                        self._ensure_worker()
+                    except Exception as exc:
+                        logger.warning(
+                            "Reranker warmup attempt %d/%d: worker spawn "
+                            "raised: %s", attempt, _WARMUP_MAX_ATTEMPTS, exc,
+                        )
+                        self._worker_proc = None
+
+                    if self._worker_proc is None:
+                        # Either the spawn failed, or another process already
+                        # owns the machine-wide singleton worker.  If a sibling
+                        # worker is alive this instance will use it on demand —
+                        # stop retrying quietly rather than spinning.
+                        if _is_reranker_worker_alive():
+                            logger.debug(
+                                "Reranker warmup: worker owned by another "
+                                "process; this instance uses it on demand",
+                            )
+                            return
+                    else:
+                        # Give the ONE-TIME model load a generous budget — it is
+                        # far heavier than a live rerank request. On timeout
+                        # _send_request kills the worker, so the next attempt
+                        # respawns cleanly (no stale-response race).
+                        resp = None
+                        try:
+                            resp = self._send_request({
+                                "cmd": "load",
+                                "model_name": self._model_name,
+                                "backend": self._backend,
+                            }, timeout=_WARMUP_LOAD_TIMEOUT)
+                        except Exception as exc:
+                            logger.warning(
+                                "Reranker warmup attempt %d/%d: load request "
+                                "raised: %s",
+                                attempt, _WARMUP_MAX_ATTEMPTS, exc,
+                            )
+                        if resp and resp.get("ok"):
+                            self._model_loaded = True
+                            logger.info(
+                                "Reranker worker warm (attempt %d/%d, "
+                                "backend=%s, warmup_inference=%s)",
+                                attempt, _WARMUP_MAX_ATTEMPTS,
+                                resp.get("backend", "?"),
+                                resp.get("warmup_inference", False),
+                            )
+                            return
+                        logger.warning(
+                            "Reranker warmup attempt %d/%d did not confirm "
+                            "ready (timeout=%ds); retrying",
+                            attempt, _WARMUP_MAX_ATTEMPTS, _WARMUP_LOAD_TIMEOUT,
+                        )
+
+                    if attempt < _WARMUP_MAX_ATTEMPTS and not self._model_loaded:
+                        time.sleep(min(_WARMUP_RETRY_BACKOFF_S * attempt, 15.0))
+
+                if not self._model_loaded:
+                    logger.warning(
+                        "Reranker warmup exhausted %d attempts; recall uses "
+                        "fallback scoring until the next rerank triggers a "
+                        "fresh load. Run 'slm doctor' for diagnostics.",
+                        _WARMUP_MAX_ATTEMPTS,
                     )
             except Exception as exc:
                 logger.debug("Background reranker warmup failed: %s", exc)
@@ -337,6 +404,12 @@ class CrossEncoderReranker:
             # Detach first so re-entrant/finalizer cleanup is idempotent.
             self._worker_proc = None
             self._worker_ready = False
+            # Invariant: a dead worker has no loaded model. Enforcing this in
+            # ONE place (not just the recycle/timeout callers) means the idle
+            # timer's kill also clears the flag, so the recall path sees the
+            # gap and triggers a background re-warmup instead of sending a
+            # rerank to a cold worker and risking a 15s-timeout churn.
+            self._model_loaded = False
             try:
                 proc.stdin.write('{"cmd":"quit"}\n')
                 proc.stdin.flush()
@@ -411,8 +484,18 @@ class CrossEncoderReranker:
         if not candidates:
             return [], False, "no_candidates"
 
-        # Non-blocking: if model isn't loaded yet, return fallback
+        # Non-blocking: if the model isn't loaded, return fallback AND kick a
+        # background (re)warmup so the reranker SELF-HEALS.  Without this, a
+        # worker that was recycled (every 500 reqs), idle-killed (30 min), or
+        # crashed left ``_model_loaded`` False with nothing to reload it — every
+        # subsequent recall degraded to fallback scoring until the daemon was
+        # restarted (a silent, sustained quality regression).  The warmup is
+        # guarded (no-op if already loading/loaded), retried, and never blocks
+        # this recall — it returns fallback now and full quality resumes within
+        # seconds once the model is warm again.
         if not self._model_loaded:
+            if not self._worker_loading:
+                self._start_background_warmup()
             sorted_cands = sorted(candidates, key=lambda x: x[1], reverse=True)
             return sorted_cands[:top_k], False, "fallback_not_ready"
 

package/src/superlocalmemory/retrieval/spreading_activation.py

@@ -224,8 +224,13 @@ class SpreadingActivation:
             if not self._fok_check(activations):
                 return []
 
-            # Cache results
-            self._cache_results(query_hash, profile_id, activations)
+            # Cache results — DEFERRED so recall stays READ-ONLY on its hot
+            # path.  This is a perf cache for FUTURE recalls, not needed to
+            # return the current results.
+            from superlocalmemory.storage.deferred_writes import submit_background
+            submit_background(
+                lambda: self._cache_results(query_hash, profile_id, activations)
+            )
 
             # Return top-K sorted by activation
             results = sorted(
@@ -545,16 +550,19 @@ class SpreadingActivation:
     ) -> None:
         """Store results in activation_cache with 1-hour TTL."""
         try:
-            for node_id, value in activations.items():
-                self._db.execute(
-                    "INSERT OR REPLACE INTO activation_cache "
-                    "(cache_id, profile_id, query_hash, node_id, "
-                    " activation_value, iteration, created_at, expires_at) "
-                    "VALUES (?, ?, ?, ?, ?, ?, datetime('now'), "
-                    "datetime('now', '+1 hour'))",
-                    (_new_id(), profile_id, query_hash, node_id, value,
-                     self._config.max_iterations),
-                )
+            # One transaction => ONE write-lock acquisition for the whole
+            # activation cache, instead of N separately-locked writes.
+            with self._db.transaction():
+                for node_id, value in activations.items():
+                    self._db.execute(
+                        "INSERT OR REPLACE INTO activation_cache "
+                        "(cache_id, profile_id, query_hash, node_id, "
+                        " activation_value, iteration, created_at, expires_at) "
+                        "VALUES (?, ?, ?, ?, ?, ?, datetime('now'), "
+                        "datetime('now', '+1 hour'))",
+                        (_new_id(), profile_id, query_hash, node_id, value,
+                         self._config.max_iterations),
+                    )
         except Exception as exc:
             logger.debug("Cache write failed: %s", exc)