superlocalmemory 3.8.2 → 3.8.5

package/src/superlocalmemory/core/ingestion_command.py

@@ -463,40 +463,141 @@ class IngestionOperationRepository:
         derivation_state: dict[str, bool] | None = None,
         last_error: str = "",
     ) -> IngestionOperation:
-        """Finish only work owned by the caller's durable lease."""
+        """Finish only work owned by the caller's durable lease.
+
+        Fix E: when transitioning to FAILED and the operation has exhausted
+        ``_MAX_AUTOMATIC_MATERIALIZATION_ATTEMPTS``, atomically INSERT a row
+        into ``dead_letter_operations`` and set ``next_retry_at=0`` (no further
+        retries scheduled).  The original row in ``ingestion_operations`` is
+        retained — the dead-letter row is a supplemental audit record, not a
+        replacement.  Both writes happen inside a single ``db.transaction()``.
+
+        NOTE (Flaw 1 from CRIT): M031 may not exist if the live database has
+        not run migration_runner against it yet.  The INSERT is therefore
+        wrapped in a try/except so that an absent table degrades gracefully to
+        the pre-3.8.4 silent-FAILED behaviour rather than breaking ingestion.
+        Operators who have run ``slm migrate`` will get the DLQ row.
+        """
         if target not in {IngestionState.COMPLETE, IngestionState.FAILED}:
             raise InvalidStateTransition(f"enriching -> {target.value}")
         current = self.get(operation_id)
-        retry_at = 0.0
-        if target is IngestionState.FAILED:
+        # Attempt count AFTER this transition = current + 1 (the UPDATE adds
+        # the delta via attempt_count+delta but here we compare the pre-update
+        # value + 1 to the cap).
+        next_attempt_count = current.attempt_count + 1
+        is_exhausted = (
+            target is IngestionState.FAILED
+            and next_attempt_count >= _MAX_AUTOMATIC_MATERIALIZATION_ATTEMPTS
+        )
+        # Fix E: exhausted → far-future retry_at so list_materializable's
+        # `next_retry_at <= now` clause never matches, excluding the
+        # dead-lettered op from the work queue permanently.
+        # 9_999_999_999 ≈ year 2286 — well beyond any reasonable operation window.
+        _NEVER_RETRY: float = 9_999_999_999.0
+        retry_at: float = _NEVER_RETRY if is_exhausted else 0.0
+        if target is IngestionState.FAILED and not is_exhausted:
             delay = min(2 ** min(max(current.attempt_count, 1), 10), 300)
             retry_at = time.time() + delay
-        rows = self.db.execute(
-            "UPDATE ingestion_operations SET state=?, "
-            "final_fact_ids_json=COALESCE(?, final_fact_ids_json), "
-            "derivation_version=COALESCE(?, derivation_version), "
-            "derivation_state_json=COALESCE(?, derivation_state_json), "
-            "lease_owner='', lease_expires_at=0, next_retry_at=?, last_error=?, "
-            "updated_at=strftime('%Y-%m-%dT%H:%M:%fZ', 'now') "
-            "WHERE operation_id=? AND state='enriching' AND lease_owner=? "
-            "RETURNING *",
-            (
-                target.value,
-                _canonical_json(list(final_fact_ids))
-                if final_fact_ids is not None
-                else None,
-                derivation_version,
-                _canonical_json(derivation_state)
-                if derivation_state is not None
-                else None,
-                retry_at,
-                last_error,
-                operation_id,
-                owner,
-            ),
-        )
-        if not rows:
-            raise InvalidStateTransition("enriching lease ownership was lost")
+
+        try:
+            with self.db.transaction():
+                rows = self.db.execute(
+                    "UPDATE ingestion_operations SET state=?, "
+                    "final_fact_ids_json=COALESCE(?, final_fact_ids_json), "
+                    "derivation_version=COALESCE(?, derivation_version), "
+                    "derivation_state_json=COALESCE(?, derivation_state_json), "
+                    "lease_owner='', lease_expires_at=0, next_retry_at=?, last_error=?, "
+                    "updated_at=strftime('%Y-%m-%dT%H:%M:%fZ', 'now') "
+                    "WHERE operation_id=? AND state='enriching' AND lease_owner=? "
+                    "RETURNING *",
+                    (
+                        target.value,
+                        _canonical_json(list(final_fact_ids))
+                        if final_fact_ids is not None
+                        else None,
+                        derivation_version,
+                        _canonical_json(derivation_state)
+                        if derivation_state is not None
+                        else None,
+                        retry_at,
+                        last_error,
+                        operation_id,
+                        owner,
+                    ),
+                )
+                if not rows:
+                    raise InvalidStateTransition(
+                        "enriching lease ownership was lost"
+                    )
+                if is_exhausted:
+                    # Fix E: INSERT dead-letter row inside the same transaction.
+                    # Failure to write (e.g. M031 not yet migrated) must NOT
+                    # abort the state-machine UPDATE — catch at the outer level.
+                    self.db.execute(
+                        "INSERT INTO dead_letter_operations "
+                        "(original_op_id, operation_type, content, "
+                        " metadata_json, error, attempt_count, "
+                        " first_attempt_at, profile_id) "
+                        "VALUES (?, 'M018', ?, ?, ?, ?, "
+                        " (SELECT unixepoch(created_at) FROM ingestion_operations "
+                        "  WHERE operation_id=?), ?)",
+                        (
+                            operation_id,
+                            current.raw_content,
+                            json.dumps(current.metadata, separators=(",", ":"))
+                            if current.metadata else None,
+                            last_error or current.last_error,
+                            next_attempt_count,
+                            operation_id,
+                            current.profile_id,
+                        ),
+                    )
+                    logger.warning(
+                        "Operation %s exhausted %d attempts — moved to dead-letter. "
+                        "Last error: %s",
+                        operation_id,
+                        next_attempt_count,
+                        last_error or current.last_error,
+                    )
+        except InvalidStateTransition:
+            raise
+        except Exception as exc:
+            # Fix E graceful-degrade: if dead-letter INSERT fails (M031 absent),
+            # fall back to the pre-3.8.4 silent-FAILED state so ingestion
+            # continues unblocked.
+            logger.error(
+                "finish_enriching failed (dead-letter path): %s — retrying "
+                "without dead-letter INSERT",
+                exc,
+            )
+            rows = self.db.execute(
+                "UPDATE ingestion_operations SET state=?, "
+                "final_fact_ids_json=COALESCE(?, final_fact_ids_json), "
+                "derivation_version=COALESCE(?, derivation_version), "
+                "derivation_state_json=COALESCE(?, derivation_state_json), "
+                "lease_owner='', lease_expires_at=0, next_retry_at=?, last_error=?, "
+                "updated_at=strftime('%Y-%m-%dT%H:%M:%fZ', 'now') "
+                "WHERE operation_id=? AND state='enriching' AND lease_owner=? "
+                "RETURNING *",
+                (
+                    target.value,
+                    _canonical_json(list(final_fact_ids))
+                    if final_fact_ids is not None
+                    else None,
+                    derivation_version,
+                    _canonical_json(derivation_state)
+                    if derivation_state is not None
+                    else None,
+                    retry_at,
+                    last_error,
+                    operation_id,
+                    owner,
+                ),
+            )
+            if not rows:
+                raise InvalidStateTransition(
+                    "enriching lease ownership was lost"
+                ) from exc
         return self._from_row(rows[0])
 
 
@@ -624,10 +725,32 @@ class IngestionCommand:
         with _materialization_lock(operation_id):
             return self._materialize_locked(operation_id)
 
-    def _materialize_locked(self, operation_id: str) -> IngestionOperation:
+    def _materialize_locked(
+        self,
+        operation_id: str,
+        *,
+        force: bool = False,
+    ) -> IngestionOperation:
         operation = self.repository.get(operation_id)
         if operation.state is IngestionState.COMPLETE:
             return operation
+        # Fix E: guard against re-attempting exhausted (dead-lettered) operations.
+        # attempt_count is incremented by claim_enriching BEFORE this method sees
+        # the new value, so the cap check uses the CURRENT (pre-claim) count.
+        # After the (cap-1)th attempt is claimed, attempt_count reaches cap-1;
+        # finish_enriching sees current.attempt_count=cap-1, next_attempt_count=cap
+        # and inserts the dead-letter row.  Any subsequent materialize() call
+        # (attempt_count already at cap-1 in FAILED state) would re-claim and
+        # insert a second dead-letter row — this guard prevents that.
+        # ``force=True`` is the operator escape hatch used by retry() — it bypasses
+        # this guard so an admin can manually re-enqueue a dead-lettered operation.
+        if (
+            not force
+            and operation.state is IngestionState.FAILED
+            and operation.attempt_count >= _MAX_AUTOMATIC_MATERIALIZATION_ATTEMPTS - 1
+        ):
+            # Already dead-lettered — return FAILED without re-claiming.
+            return operation
         if operation.state not in {
             IngestionState.QUERYABLE,
             IngestionState.ENRICHING,
@@ -757,9 +880,15 @@ class IngestionCommand:
             )
 
     def retry(self, operation_id: str) -> IngestionOperation:
+        """Operator escape hatch: force-retry a FAILED operation regardless of
+        attempt_count.  Bypasses the Fix E dead-letter guard so an admin can
+        manually re-enqueue an exhausted operation after configuration repair.
+        """
         operation = self.repository.get(operation_id)
         if operation.state is not IngestionState.FAILED:
             raise InvalidStateTransition(
                 f"cannot retry operation in {operation.state.value}"
             )
-        return self.materialize(operation_id)
+        # force=True bypasses the _MAX_AUTOMATIC_MATERIALIZATION_ATTEMPTS guard.
+        with _materialization_lock(operation_id):
+            return self._materialize_locked(operation_id, force=True)

package/src/superlocalmemory/core/maintenance_scheduler.py

@@ -60,17 +60,42 @@ class MaintenanceScheduler:
             return
         self._running = True
         self._schedule_next()
+        # v3.8.5: one-shot activation-cache GC ~90s after boot so an upgrade
+        # backlog (observed 83k expired rows) clears promptly instead of waiting
+        # a full interval — delayed past boot warmup so it never competes for
+        # the write lock during the startup window.
+        self._initial_gc_timer = threading.Timer(90.0, self._initial_cache_gc)
+        self._initial_gc_timer.daemon = True
+        self._initial_gc_timer.start()
         logger.info(
             "Maintenance scheduler started (interval=%dm)",
             self._config.forgetting.scheduler_interval_minutes,
         )
 
+    def _initial_cache_gc(self) -> None:
+        """Best-effort one-shot activation-cache GC shortly after boot."""
+        if not self._running:
+            return
+        try:
+            deleted = self._db.cleanup_activation_cache()
+            if deleted > 0:
+                logger.info(
+                    "Activation-cache GC (startup): %d expired rows removed",
+                    deleted,
+                )
+        except Exception as exc:
+            logger.debug("Startup activation-cache GC skipped: %s", exc)
+
     def stop(self) -> None:
         """Stop the scheduler. Idempotent."""
         self._running = False
         if self._timer is not None:
             self._timer.cancel()
             self._timer = None
+        _gc_timer = getattr(self, "_initial_gc_timer", None)
+        if _gc_timer is not None:
+            _gc_timer.cancel()
+            self._initial_gc_timer = None
         logger.info("Maintenance scheduler stopped")
 
     def _schedule_next(self) -> None:
@@ -125,9 +150,33 @@ class MaintenanceScheduler:
                     )
 
             # V3.4.11: Graph pruning (remove orphan edges)
+            # v3.8.4-G: thread GraphPruningConfig params so dashboard changes
+            # persist and take effect without a daemon restart.
             try:
                 from superlocalmemory.core.graph_pruner import prune_graph
-                prune_stats = prune_graph(self._db.db_path, profile_id)
+                gp = self._config.graph_pruning
+                if not gp.enabled:
+                    logger.debug(
+                        "Graph pruning disabled by config for %s — skipping",
+                        profile_id,
+                    )
+                else:
+                    # Fix A: pass DatabaseManager directly → writes serialised through _lock
+                    prune_stats = prune_graph(
+                        self._db,
+                        profile_id,
+                        max_degree=gp.max_degree_per_node,
+                        min_edge_weight=gp.min_edge_weight,
+                    )
+                    removed = prune_stats["total_before"] - prune_stats["total_after"]
+                    if removed > 0:
+                        logger.info(
+                            "Graph pruning for %s: %d edges removed", profile_id, removed
+                        )
+            except AttributeError:
+                # Older SLMConfig without graph_pruning field (upgrade safety)
+                from superlocalmemory.core.graph_pruner import prune_graph
+                prune_stats = prune_graph(self._db, profile_id)
                 removed = prune_stats["total_before"] - prune_stats["total_after"]
                 if removed > 0:
                     logger.info("Graph pruning for %s: %d edges removed", profile_id, removed)
@@ -171,6 +220,17 @@ class MaintenanceScheduler:
         except Exception as exc:
             logger.debug("Pending cleanup skipped: %s", exc)
 
+        # v3.8.5: GC the spreading-activation result cache. Neither cleanup path
+        # was ever wired in, so activation_cache grew without bound (observed
+        # 83k expired rows, oldest ~3.5 months). Batched + DB-wide (expiry is
+        # not profile-scoped), so it runs once per cycle rather than per profile.
+        try:
+            deleted = self._db.cleanup_activation_cache()
+            if deleted > 0:
+                logger.info("Activation-cache GC: %d expired rows removed", deleted)
+        except Exception as exc:
+            logger.debug("Activation-cache GC skipped: %s", exc)
+
         self._schedule_next()
 
     def _profile_ids(self) -> tuple[str, ...]:

package/src/superlocalmemory/core/recall_pipeline.py

@@ -776,6 +776,9 @@ def run_recall(
     if access_log and response.results:
         try:
             fact_ids = [r.fact.fact_id for r in response.results]
+            # Recall exposure logging is a durable analytics contract (tested),
+            # so it stays synchronous — it is a single batched write, cheap
+            # relative to the deferred spreading-activation cache / last_seen.
             access_log.store_access_batch(
                 fact_ids=fact_ids,
                 profile_id=profile_id,

package/src/superlocalmemory/core/registry.py

@@ -110,7 +110,11 @@ class AgentRegistry:
         try:
             data = json.loads(self._path.read_text(encoding="utf-8"))
             self._agents = data.get("agents", {})
-            self._write_locks = data.get("write_locks", {})
+            # D-01: write_locks from a prior process are meaningless after a
+            # process boundary — the agents that held them are dead.  Starting
+            # with no inherited locks prevents mcp_client (and any other agent)
+            # from being permanently blocked on every daemon restart.
+            self._write_locks = {}
         except (json.JSONDecodeError, KeyError):
             logger.warning("Corrupt registry file at %s — starting fresh.", self._path)
             self._agents = {}

package/src/superlocalmemory/core/remote_mode.py

@@ -182,7 +182,9 @@ def is_rate_limit_exempt(client_host: str) -> bool:
     same rapid reads, so it is exempt too — otherwise normal dashboard polling
     trips the limiter (issue #40 Issue 3).
     """
-    if client_host in ("127.0.0.1", "::1", "localhost"):
+    from superlocalmemory.server.loopback import is_loopback as _is_loopback_host
+
+    if _is_loopback_host(client_host):
         return True
     return is_lan_client_allowed(client_host)
 

package/src/superlocalmemory/core/scale_engine.py

@@ -349,7 +349,24 @@ class ScaleEngineManager:
             self._release_lifecycle_lock(lock_path)
 
     def _promote(self, stage_id: str) -> dict[str, Any]:
-        """Promote while the caller owns the lifecycle lock."""
+        """Promote while the caller owns the lifecycle lock.
+
+        Concurrency fix (v3.8.4): the original code held a SQLite
+        BEGIN IMMEDIATE lock across multiple fsync/rename filesystem operations
+        (mkdir_durable, replace_durable, write_promotion_journal) which could
+        starve other memory.db writers for 30+ seconds.
+
+        Fix: the fingerprint check runs inside a short memory_read() block
+        (consistent WAL snapshot, no write lock).  All filesystem operations
+        happen AFTER that block with no SQLite lock held.  The lifecycle lock
+        (file-based, acquired by the caller) already serialises concurrent
+        promote/rollback calls.
+
+        Lock-ordering invariant preserved: get_write_lock is OUTERMOST; no
+        SQLite write lock is held here at all (only a read snapshot).
+        """
+        from superlocalmemory.storage.memory_write import memory_read
+
         stage_dir, manifest = self._load_stage(stage_id)
         self._validate_manifest(manifest, state="verified")
         staged = (stage_dir / "cozo", stage_dir / "lance")
@@ -357,17 +374,30 @@ class ScaleEngineManager:
             raise ScaleEngineError("verified stage is incomplete; prepare a new stage")
         backup_dir = self.backup_root / f"{stage_id}-{uuid.uuid4().hex[:6]}"
         active = self.active_paths
-        gate = sqlite3.connect(self.db_path, timeout=30)
+
+        # ── Phase 1: fingerprint check (short read snapshot, no write lock) ──
+        # memory_read() gives a consistent WAL snapshot and closes the
+        # connection immediately on exit — no lock held after this block.
+        try:
+            with memory_read(self.db_path) as gate:
+                canonical = self._canonical_counts(gate)
+                if manifest["source_fingerprint"] != self._projection_fingerprint(
+                    gate, canonical
+                ):
+                    raise ScaleEngineError(
+                        "canonical SQLite changed after verification; prepare a new stage"
+                    )
+        except ScaleEngineError:
+            raise
+        except Exception as exc:
+            raise ScaleEngineError(
+                f"fingerprint check failed: {exc}"
+            ) from exc
+
+        # ── Phase 2: filesystem operations — NO SQLite lock held ─────────────
+        # The lifecycle lock (file-based, held by the caller) serialises
+        # concurrent promotions; no SQLite fence is required for the swaps.
         try:
-            # The stage was built from a point-in-time SQLite snapshot. Hold a
-            # short writer fence for the final fingerprint check and directory
-            # swap so no successful promotion can trail a canonical write.
-            gate.execute("BEGIN IMMEDIATE")
-            canonical = self._canonical_counts(gate)
-            if manifest["source_fingerprint"] != self._projection_fingerprint(gate, canonical):
-                raise ScaleEngineError(
-                    "canonical SQLite changed after verification; prepare a new stage"
-                )
             self._mkdir_durable(self.backup_root)
             journal = {
                 "schema_version": self.SCHEMA_VERSION,
@@ -409,13 +439,8 @@ class ScaleEngineManager:
             journal["state"] = "committed"
             self._write_promotion_journal(journal)
             self.promotion_journal_path.unlink(missing_ok=True)
-            gate.rollback()
             return manifest
         except Exception as exc:
-            try:
-                gate.rollback()
-            except sqlite3.Error:
-                pass
             try:
                 recovery = self._recover_interrupted_promotion()
             except Exception as recovery_error:
@@ -426,8 +451,6 @@ class ScaleEngineManager:
                 _, recovered_manifest = self._load_stage(stage_id)
                 return recovered_manifest
             raise ScaleEngineError(f"promotion rolled back: {exc}") from exc
-        finally:
-            gate.close()
 
     def rollback(self, backup_id: str) -> dict[str, Any]:
         """Restore an explicitly named pre-promotion backup."""

package/src/superlocalmemory/core/store_pipeline.py

@@ -166,10 +166,24 @@ def enrich_fact(
     from superlocalmemory.encoding.emotional import emotional_importance_boost, tag_emotion
     from superlocalmemory.encoding.signal_inference import infer_signal
 
-    embedding = embedder.embed(fact.content) if embedder else None
-    fisher_mean, fisher_variance = (None, None)
-    if embedder and embedding:
-        fisher_mean, fisher_variance = embedder.compute_fisher_params(embedding)
+    # v3.8.4 D: if the fact already carries a sync-embedded vector (warm-guard
+    # path in store_fast), reuse it — avoids a redundant embed call in the
+    # materializer and keeps the vector consistent with the one indexed in the
+    # vector store at write time.
+    if fact.embedding is not None:
+        embedding = fact.embedding
+        fisher_mean = fact.fisher_mean
+        fisher_variance = fact.fisher_variance
+        # fisher_params are computed in the warm-guard path too, but guard
+        # against the edge case where they weren't (e.g. compute_fisher_params
+        # raised after embed succeeded).
+        if (fisher_mean is None or fisher_variance is None) and embedder and embedding:
+            fisher_mean, fisher_variance = embedder.compute_fisher_params(embedding)
+    else:
+        embedding = embedder.embed(fact.content) if embedder else None
+        fisher_mean, fisher_variance = (None, None)
+        if embedder and embedding:
+            fisher_mean, fisher_variance = embedder.compute_fisher_params(embedding)
 
     canonical = {}
     if entity_resolver and fact.entities:

package/src/superlocalmemory/encoding/entity_resolver.py

@@ -563,18 +563,25 @@ class EntityResolver:
         self._db.store_alias(alias, profile_id)
 
     def _touch_last_seen(self, entity_id: str, profile_id: str = "default") -> None:
-        """Update last_seen timestamp on a canonical entity scoped to profile.
-
-        L-01 fix: the original query had no profile_id guard. Since entity_ids are
-        UUIDs today the blast radius is theoretical, but the guard is required for
-        defense-in-depth against future import/sharing features that could introduce
-        UUID collisions across profiles.
+        """Record a last_seen touch (DEFERRED, non-blocking).
+
+        last_seen is dashboard-only bookkeeping (entities/graph "last seen"
+        columns); it never feeds recall ranking.  Writing it INLINE made recall
+        a WRITER that waited on the global write lock — the root of the
+        "recall is 8 s" regression.  We now record it in memory and flush from a
+        single background thread in coalesced batches, so recall stays
+        READ-ONLY on its hot path.  The dashboard stays correct within the flush
+        interval (~2 s).  The profile_id guard (L-01) is preserved in the
+        deferred UPDATE (see storage/deferred_writes.py).
         """
-        self._db.execute(
-            "UPDATE canonical_entities SET last_seen = ? "
-            "WHERE entity_id = ? AND profile_id = ?",
-            (_now(), entity_id, profile_id),
-        )
+        try:
+            from superlocalmemory.storage.deferred_writes import (
+                get_deferred_last_seen,
+            )
+            get_deferred_last_seen(self._db).touch(entity_id, profile_id, _now())
+        except Exception:
+            # Bookkeeping must never break entity resolution.
+            pass
 
     # -- Internal: LLM disambiguation (Mode B/C) ---------------------------
 

package/src/superlocalmemory/hooks/_outcome_common.py

@@ -40,8 +40,15 @@ from typing import IO, Optional
 # Budget constants
 # ---------------------------------------------------------------------------
 
-#: Hot-path SQLite busy timeout (ms). Fail fast rather than block a host tool.
-BUSY_TIMEOUT_MS: int = 50
+#: Hot-path SQLite busy timeout (ms).
+#
+# Raised to 10 000 ms to match the daemon's SLM_DB_BUSY_TIMEOUT_MS default.
+# Hooks run as a SEPARATE OS process — the daemon's threading.RLock write-lock
+# cannot help cross-process — so PRAGMA busy_timeout is the ONLY lever that
+# prevents SQLITE_BUSY when the daemon or CLI holds the WAL write lock.
+# 10 s is long enough to outlast a typical daemon write cycle while staying
+# safely below Claude Code's hook-kill timeout.
+BUSY_TIMEOUT_MS: int = 10_000
 
 #: Cap on tool_response bytes scanned — bounds substring work to O(100 KB).
 SCAN_BYTES_CAP: int = 100_000