superlocalmemory 3.8.3 → 3.8.6

package/src/superlocalmemory/core/ingestion_command.py

@@ -13,6 +13,7 @@ from __future__ import annotations
 
 import hashlib
 import json
+import logging
 import sqlite3
 import threading
 import time
@@ -21,14 +22,13 @@ from dataclasses import dataclass, field
 from enum import Enum
 from typing import Any, Callable
 
-import logging
-
 from superlocalmemory.storage.database import DatabaseManager
 
 logger = logging.getLogger("superlocalmemory.ingestion_command")
 
 _MATERIALIZATION_LOCKS = tuple(threading.RLock() for _ in range(64))
 _MAX_AUTOMATIC_MATERIALIZATION_ATTEMPTS = 10
+_NEVER_RETRY_AT = 9_999_999_999.0
 
 
 def _materialization_lock(operation_id: str) -> threading.RLock:
@@ -48,6 +48,10 @@ class IdempotencyConflict(ValueError):
     """The same idempotency key was reused for different immutable evidence."""
 
 
+class IngestionRejectedError(RuntimeError):
+    """Deterministic evidence policy produced no queryable projection."""
+
+
 class InvalidStateTransition(RuntimeError):
     """An ingestion operation attempted an illegal or stale transition."""
 
@@ -463,41 +467,245 @@ class IngestionOperationRepository:
         derivation_state: dict[str, bool] | None = None,
         last_error: str = "",
     ) -> IngestionOperation:
-        """Finish only work owned by the caller's durable lease."""
+        """Finish only work owned by the caller's durable lease.
+
+        Fix E: when transitioning to FAILED and the operation has exhausted
+        ``_MAX_AUTOMATIC_MATERIALIZATION_ATTEMPTS``, atomically INSERT a row
+        into ``dead_letter_operations`` and set ``next_retry_at=0`` (no further
+        retries scheduled).  The original row in ``ingestion_operations`` is
+        retained — the dead-letter row is a supplemental audit record, not a
+        replacement.  Both writes happen inside a single ``db.transaction()``.
+
+        NOTE (Flaw 1 from CRIT): M031 may not exist if the live database has
+        not run migration_runner against it yet.  The INSERT is therefore
+        wrapped in a try/except so that an absent table degrades gracefully to
+        the pre-3.8.4 silent-FAILED behaviour rather than breaking ingestion.
+        Operators who have run ``slm migrate`` will get the DLQ row.
+        """
         if target not in {IngestionState.COMPLETE, IngestionState.FAILED}:
             raise InvalidStateTransition(f"enriching -> {target.value}")
         current = self.get(operation_id)
-        retry_at = 0.0
-        if target is IngestionState.FAILED:
+        # claim_enriching() already increments attempt_count before the
+        # materializer runs.  finish_enriching() records that claimed attempt;
+        # incrementing again here would dead-letter after only nine real tries
+        # while claiming that ten had run.
+        attempt_count = current.attempt_count
+        is_exhausted = (
+            target is IngestionState.FAILED
+            and attempt_count >= _MAX_AUTOMATIC_MATERIALIZATION_ATTEMPTS
+        )
+        # Fix E: exhausted → far-future retry_at so list_materializable's
+        # `next_retry_at <= now` clause never matches, excluding the
+        # dead-lettered op from the work queue permanently.
+        # 9_999_999_999 ≈ year 2286 — well beyond any reasonable operation window.
+        _NEVER_RETRY: float = 9_999_999_999.0
+        retry_at: float = _NEVER_RETRY if is_exhausted else 0.0
+        if target is IngestionState.FAILED and not is_exhausted:
             delay = min(2 ** min(max(current.attempt_count, 1), 10), 300)
             retry_at = time.time() + delay
-        rows = self.db.execute(
-            "UPDATE ingestion_operations SET state=?, "
-            "final_fact_ids_json=COALESCE(?, final_fact_ids_json), "
-            "derivation_version=COALESCE(?, derivation_version), "
-            "derivation_state_json=COALESCE(?, derivation_state_json), "
-            "lease_owner='', lease_expires_at=0, next_retry_at=?, last_error=?, "
-            "updated_at=strftime('%Y-%m-%dT%H:%M:%fZ', 'now') "
-            "WHERE operation_id=? AND state='enriching' AND lease_owner=? "
-            "RETURNING *",
+
+        try:
+            with self.db.transaction():
+                rows = self.db.execute(
+                    "UPDATE ingestion_operations SET state=?, "
+                    "final_fact_ids_json=COALESCE(?, final_fact_ids_json), "
+                    "derivation_version=COALESCE(?, derivation_version), "
+                    "derivation_state_json=COALESCE(?, derivation_state_json), "
+                    "lease_owner='', lease_expires_at=0, next_retry_at=?, last_error=?, "
+                    "updated_at=strftime('%Y-%m-%dT%H:%M:%fZ', 'now') "
+                    "WHERE operation_id=? AND state='enriching' AND lease_owner=? "
+                    "RETURNING *",
+                    (
+                        target.value,
+                        _canonical_json(list(final_fact_ids))
+                        if final_fact_ids is not None
+                        else None,
+                        derivation_version,
+                        _canonical_json(derivation_state)
+                        if derivation_state is not None
+                        else None,
+                        retry_at,
+                        last_error,
+                        operation_id,
+                        owner,
+                    ),
+                )
+                if not rows:
+                    raise InvalidStateTransition(
+                        "enriching lease ownership was lost"
+                    )
+                if is_exhausted:
+                    # Fix E: INSERT dead-letter row inside the same transaction.
+                    # Failure to write (e.g. M031 not yet migrated) must NOT
+                    # abort the state-machine UPDATE — catch at the outer level.
+                    self.db.execute(
+                        "INSERT INTO dead_letter_operations "
+                        "(original_op_id, operation_type, content, "
+                        " metadata_json, error, attempt_count, "
+                        " first_attempt_at, dead_lettered_at, profile_id) "
+                        "VALUES (?, 'M018', ?, ?, ?, ?, "
+                        " (SELECT CAST(strftime('%s', created_at) AS REAL) "
+                        "  FROM ingestion_operations WHERE operation_id=?), ?, ?)",
+                        (
+                            operation_id,
+                            current.raw_content,
+                            json.dumps(current.metadata, separators=(",", ":"))
+                            if current.metadata else None,
+                            last_error or current.last_error,
+                            attempt_count,
+                            operation_id,
+                            time.time(),
+                            current.profile_id,
+                        ),
+                    )
+                    logger.warning(
+                        "Operation %s exhausted %d attempts — moved to dead-letter. "
+                        "Last error: %s",
+                        operation_id,
+                        attempt_count,
+                        last_error or current.last_error,
+                    )
+        except InvalidStateTransition:
+            raise
+        except Exception as exc:
+            # Fix E graceful-degrade: if dead-letter INSERT fails (M031 absent),
+            # fall back to the pre-3.8.4 silent-FAILED state so ingestion
+            # continues unblocked.
+            logger.error(
+                "finish_enriching failed (dead-letter path): %s — retrying "
+                "without dead-letter INSERT",
+                exc,
+            )
+            rows = self.db.execute(
+                "UPDATE ingestion_operations SET state=?, "
+                "final_fact_ids_json=COALESCE(?, final_fact_ids_json), "
+                "derivation_version=COALESCE(?, derivation_version), "
+                "derivation_state_json=COALESCE(?, derivation_state_json), "
+                "lease_owner='', lease_expires_at=0, next_retry_at=?, last_error=?, "
+                "updated_at=strftime('%Y-%m-%dT%H:%M:%fZ', 'now') "
+                "WHERE operation_id=? AND state='enriching' AND lease_owner=? "
+                "RETURNING *",
+                (
+                    target.value,
+                    _canonical_json(list(final_fact_ids))
+                    if final_fact_ids is not None
+                    else None,
+                    derivation_version,
+                    _canonical_json(derivation_state)
+                    if derivation_state is not None
+                    else None,
+                    retry_at,
+                    last_error,
+                    operation_id,
+                    owner,
+                ),
+            )
+            if not rows:
+                raise InvalidStateTransition(
+                    "enriching lease ownership was lost"
+                ) from exc
+        return self._from_row(rows[0])
+
+    def reap_stuck_enriching(
+        self,
+        *,
+        now: float | None = None,
+        limit: int = 100,
+    ) -> list[str]:
+        """Terminalize expired enrichment leases that exhausted automatic retries.
+
+        ``list_materializable`` intentionally excludes operations at the retry
+        cap.  If a worker dies while such an operation is still ``enriching``,
+        it otherwise becomes a permanent phantom: no worker can reclaim it and
+        it never reaches a terminal state.  Queryable facts are already durable,
+        so reaping abandons only optional derivation work.
+
+        Each candidate is transitioned with a compare-and-swap update in its
+        own bounded transaction.  A dead-letter record is supplemental; an
+        older database without M031 is still terminalized safely.
+        """
+        cutoff = time.time() if now is None else float(now)
+        batch_limit = max(1, min(int(limit), 500))
+        candidates = self.db.execute(
+            "SELECT operation_id, attempt_count, last_error, raw_content, "
+            "raw_metadata_json, profile_id FROM ingestion_operations "
+            "WHERE state='enriching' AND lease_expires_at <= ? "
+            "AND attempt_count >= ? ORDER BY updated_at, operation_id LIMIT ?",
             (
-                target.value,
-                _canonical_json(list(final_fact_ids))
-                if final_fact_ids is not None
-                else None,
-                derivation_version,
-                _canonical_json(derivation_state)
-                if derivation_state is not None
-                else None,
-                retry_at,
-                last_error,
-                operation_id,
-                owner,
+                cutoff,
+                _MAX_AUTOMATIC_MATERIALIZATION_ATTEMPTS,
+                batch_limit,
             ),
         )
-        if not rows:
-            raise InvalidStateTransition("enriching lease ownership was lost")
-        return self._from_row(rows[0])
+        reaped: list[str] = []
+        for row in candidates:
+            data = dict(row)
+            operation_id = str(data["operation_id"])
+            terminal_error = (
+                data["last_error"]
+                or "reaped: enrichment exhausted automatic attempts"
+            )
+            try:
+                with self.db.transaction():
+                    updated = self.db.execute(
+                        "UPDATE ingestion_operations SET state='failed', "
+                        "lease_owner='', lease_expires_at=0, next_retry_at=?, "
+                        "last_error=?, "
+                        "updated_at=strftime('%Y-%m-%dT%H:%M:%fZ', 'now') "
+                        "WHERE operation_id=? AND state='enriching' "
+                        "AND lease_expires_at <= ? AND attempt_count >= ? "
+                        "RETURNING operation_id",
+                        (
+                            _NEVER_RETRY_AT,
+                            terminal_error,
+                            operation_id,
+                            cutoff,
+                            _MAX_AUTOMATIC_MATERIALIZATION_ATTEMPTS,
+                        ),
+                    )
+                    if not updated:
+                        continue
+                    try:
+                        self.db.execute(
+                            "INSERT INTO dead_letter_operations "
+                            "(original_op_id, operation_type, content, "
+                            "metadata_json, error, attempt_count, "
+                            "first_attempt_at, dead_lettered_at, profile_id) "
+                            "VALUES (?, 'M018', ?, ?, ?, ?, "
+                            "(SELECT CAST(strftime('%s', created_at) AS REAL) "
+                            "FROM ingestion_operations WHERE operation_id=?), ?, ?)",
+                            (
+                                operation_id,
+                                data["raw_content"],
+                                data["raw_metadata_json"] or None,
+                                terminal_error,
+                                int(data["attempt_count"]),
+                                operation_id,
+                                time.time(),
+                                data["profile_id"],
+                            ),
+                        )
+                    except sqlite3.OperationalError as exc:
+                        if "no such table" not in str(exc).lower():
+                            raise
+                        logger.info(
+                            "Dead-letter table unavailable while reaping %s; "
+                            "terminal transition preserved",
+                            operation_id,
+                        )
+            except Exception:
+                logger.exception(
+                    "Failed to reap exhausted ingestion operation %s",
+                    operation_id,
+                )
+                continue
+            reaped.append(operation_id)
+            logger.warning(
+                "Reaped exhausted ingestion operation %s at attempt %d",
+                operation_id,
+                int(data["attempt_count"]),
+            )
+        return reaped
 
 
 @dataclass(frozen=True, slots=True)
@@ -510,6 +718,7 @@ class MaterializationResult:
 
 
 QueryableWriter = Callable[[IngestionRequest, str], list[str]]
+AdmissionValidator = Callable[[IngestionRequest], None]
 Materializer = Callable[
     [IngestionOperation],
     list[str] | tuple[str, ...] | MaterializationResult,
@@ -526,6 +735,7 @@ class IngestionCommand:
         *,
         write_queryable: QueryableWriter,
         materialize: Materializer,
+        validate_admission: AdmissionValidator | None = None,
         project: Projector | None = None,
         derivation_version: str = "v3.7-ingestion-1",
         lease_seconds: float = 900.0,
@@ -533,6 +743,7 @@ class IngestionCommand:
         self.repository = repository
         self._write_queryable = write_queryable
         self._materializer = materialize
+        self._validate_admission = validate_admission
         self._projector = project
         self._derivation_version = derivation_version
         self._lease_seconds = max(1.0, float(lease_seconds))
@@ -603,13 +814,18 @@ class IngestionCommand:
         self, request: IngestionRequest,
     ) -> tuple[IngestionOperation, bool]:
         """Submit once and report whether this call created the operation."""
+        # Trust/authentication and deterministic policy checks belong before
+        # the durable transaction.  They may reject, log, or consult policy,
+        # but must never extend SQLite's writer critical section.
+        if self._validate_admission is not None:
+            self._validate_admission(request)
         with self.repository.db.transaction():
             operation, created = self.repository.create_with_status(request)
             if operation.state is not IngestionState.RAW:
                 return operation, created
             fact_ids = tuple(self._write_queryable(request, operation.operation_id))
             if not fact_ids:
-                raise RuntimeError("ingestion produced no queryable facts")
+                raise IngestionRejectedError("ingestion produced no queryable facts")
             receipt = self.repository.transition(
                 operation.operation_id,
                 expected=IngestionState.RAW,
@@ -624,10 +840,29 @@ class IngestionCommand:
         with _materialization_lock(operation_id):
             return self._materialize_locked(operation_id)
 
-    def _materialize_locked(self, operation_id: str) -> IngestionOperation:
+    def _materialize_locked(
+        self,
+        operation_id: str,
+        *,
+        force: bool = False,
+    ) -> IngestionOperation:
         operation = self.repository.get(operation_id)
         if operation.state is IngestionState.COMPLETE:
             return operation
+        # Fix E: guard against re-attempting exhausted (dead-lettered) operations.
+        # claim_enriching increments the count before each real attempt, and
+        # finish_enriching dead-letters the failure whose count reaches the
+        # cap. A subsequent materialize() call sees count==cap and must not
+        # re-claim or insert another dead-letter row.
+        # ``force=True`` is the operator escape hatch used by retry() — it bypasses
+        # this guard so an admin can manually re-enqueue a dead-lettered operation.
+        if (
+            not force
+            and operation.state is IngestionState.FAILED
+            and operation.attempt_count >= _MAX_AUTOMATIC_MATERIALIZATION_ATTEMPTS
+        ):
+            # Already dead-lettered — return FAILED without re-claiming.
+            return operation
         if operation.state not in {
             IngestionState.QUERYABLE,
             IngestionState.ENRICHING,
@@ -757,9 +992,15 @@ class IngestionCommand:
             )
 
     def retry(self, operation_id: str) -> IngestionOperation:
+        """Operator escape hatch: force-retry a FAILED operation regardless of
+        attempt_count.  Bypasses the Fix E dead-letter guard so an admin can
+        manually re-enqueue an exhausted operation after configuration repair.
+        """
         operation = self.repository.get(operation_id)
         if operation.state is not IngestionState.FAILED:
             raise InvalidStateTransition(
                 f"cannot retry operation in {operation.state.value}"
             )
-        return self.materialize(operation_id)
+        # force=True bypasses the _MAX_AUTOMATIC_MATERIALIZATION_ATTEMPTS guard.
+        with _materialization_lock(operation_id):
+            return self._materialize_locked(operation_id, force=True)

package/src/superlocalmemory/core/maintenance_scheduler.py

@@ -60,17 +60,42 @@ class MaintenanceScheduler:
             return
         self._running = True
         self._schedule_next()
+        # v3.8.5: one-shot activation-cache GC ~90s after boot so an upgrade
+        # backlog (observed 83k expired rows) clears promptly instead of waiting
+        # a full interval — delayed past boot warmup so it never competes for
+        # the write lock during the startup window.
+        self._initial_gc_timer = threading.Timer(90.0, self._initial_cache_gc)
+        self._initial_gc_timer.daemon = True
+        self._initial_gc_timer.start()
         logger.info(
             "Maintenance scheduler started (interval=%dm)",
             self._config.forgetting.scheduler_interval_minutes,
         )
 
+    def _initial_cache_gc(self) -> None:
+        """Best-effort one-shot activation-cache GC shortly after boot."""
+        if not self._running:
+            return
+        try:
+            deleted = self._db.cleanup_activation_cache()
+            if deleted > 0:
+                logger.info(
+                    "Activation-cache GC (startup): %d expired rows removed",
+                    deleted,
+                )
+        except Exception as exc:
+            logger.debug("Startup activation-cache GC skipped: %s", exc)
+
     def stop(self) -> None:
         """Stop the scheduler. Idempotent."""
         self._running = False
         if self._timer is not None:
             self._timer.cancel()
             self._timer = None
+        _gc_timer = getattr(self, "_initial_gc_timer", None)
+        if _gc_timer is not None:
+            _gc_timer.cancel()
+            self._initial_gc_timer = None
         logger.info("Maintenance scheduler stopped")
 
     def _schedule_next(self) -> None:
@@ -125,9 +150,33 @@ class MaintenanceScheduler:
                     )
 
             # V3.4.11: Graph pruning (remove orphan edges)
+            # v3.8.4-G: thread GraphPruningConfig params so dashboard changes
+            # persist and take effect without a daemon restart.
             try:
                 from superlocalmemory.core.graph_pruner import prune_graph
-                prune_stats = prune_graph(self._db.db_path, profile_id)
+                gp = self._config.graph_pruning
+                if not gp.enabled:
+                    logger.debug(
+                        "Graph pruning disabled by config for %s — skipping",
+                        profile_id,
+                    )
+                else:
+                    # Fix A: pass DatabaseManager directly → writes serialised through _lock
+                    prune_stats = prune_graph(
+                        self._db,
+                        profile_id,
+                        max_degree=gp.max_degree_per_node,
+                        min_edge_weight=gp.min_edge_weight,
+                    )
+                    removed = prune_stats["total_before"] - prune_stats["total_after"]
+                    if removed > 0:
+                        logger.info(
+                            "Graph pruning for %s: %d edges removed", profile_id, removed
+                        )
+            except AttributeError:
+                # Older SLMConfig without graph_pruning field (upgrade safety)
+                from superlocalmemory.core.graph_pruner import prune_graph
+                prune_stats = prune_graph(self._db, profile_id)
                 removed = prune_stats["total_before"] - prune_stats["total_after"]
                 if removed > 0:
                     logger.info("Graph pruning for %s: %d edges removed", profile_id, removed)
@@ -171,6 +220,17 @@ class MaintenanceScheduler:
         except Exception as exc:
             logger.debug("Pending cleanup skipped: %s", exc)
 
+        # v3.8.5: GC the spreading-activation result cache. Neither cleanup path
+        # was ever wired in, so activation_cache grew without bound (observed
+        # 83k expired rows, oldest ~3.5 months). Batched + DB-wide (expiry is
+        # not profile-scoped), so it runs once per cycle rather than per profile.
+        try:
+            deleted = self._db.cleanup_activation_cache()
+            if deleted > 0:
+                logger.info("Activation-cache GC: %d expired rows removed", deleted)
+        except Exception as exc:
+            logger.debug("Activation-cache GC skipped: %s", exc)
+
         self._schedule_next()
 
     def _profile_ids(self) -> tuple[str, ...]:

package/src/superlocalmemory/core/mutations.py

@@ -42,6 +42,8 @@ def delete_fact_authorized(
     *,
     trusted_actor_id: str,
     source_agent_id: str,
+    canonical_runtime: Any | None = None,
+    idempotency_key: str | None = None,
 ) -> dict[str, Any]:
     """Authorize, delete one profile-owned fact, then emit post hooks."""
     profile_id, context = _context(
@@ -51,15 +53,23 @@ def delete_fact_authorized(
         trusted_actor_id=trusted_actor_id,
         source_agent_id=source_agent_id,
     )
-    rows = engine._db.execute(
-        "SELECT content FROM atomic_facts "
-        "WHERE fact_id = ? AND profile_id = ? LIMIT 1",
-        (fact_id, profile_id),
-    )
-    if not rows:
-        return {"ok": False, "error": f"Memory {fact_id} not found"}
-    content_preview = dict(rows[0]).get("content", "")[:80]
-    engine._db.delete_fact(fact_id, profile_id=profile_id)
+    if canonical_runtime is not None:
+        result = dict(canonical_runtime.delete_fact(
+            profile_id, fact_id, idempotency_key=idempotency_key,
+        ))
+        if not result.get("ok"):
+            return {"ok": False, "error": f"Memory {fact_id} not found"}
+        content_preview = str(result.get("content_preview", ""))
+    else:
+        rows = engine._db.execute(
+            "SELECT content FROM atomic_facts "
+            "WHERE fact_id = ? AND profile_id = ? LIMIT 1",
+            (fact_id, profile_id),
+        )
+        if not rows:
+            return {"ok": False, "error": f"Memory {fact_id} not found"}
+        content_preview = dict(rows[0]).get("content", "")[:80]
+        engine._db.delete_fact(fact_id, profile_id=profile_id)
     try:
         from superlocalmemory.core.backend_orchestrator import get_orchestrator
         orchestrator = get_orchestrator()
@@ -86,6 +96,8 @@ def update_fact_authorized(
     *,
     trusted_actor_id: str,
     source_agent_id: str,
+    canonical_runtime: Any | None = None,
+    idempotency_key: str | None = None,
 ) -> dict[str, Any]:
     """Authorize a fact update and refresh semantic and lexical indexes."""
     if not content or not content.strip():
@@ -120,7 +132,17 @@ def update_fact_authorized(
                 updates["fisher_variance"] = fisher_variance
         except Exception as exc:
             logger.warning("UPDATE embedding refresh failed: %s", exc)
-    engine._db.update_fact(fact_id, updates, profile_id=profile_id)
+    if canonical_runtime is not None:
+        result = dict(canonical_runtime.update_fact(
+            profile_id,
+            fact_id,
+            updates,
+            idempotency_key=idempotency_key,
+        ))
+        if not result.get("ok"):
+            return {"ok": False, "error": f"Memory {fact_id} not found"}
+    else:
+        engine._db.update_fact(fact_id, updates, profile_id=profile_id)
     try:
         from superlocalmemory.core.backend_orchestrator import get_orchestrator
         orchestrator = get_orchestrator()