superlocalmemory 3.8.5 → 3.8.7

package/src/superlocalmemory/core/ingestion_command.py

@@ -13,6 +13,7 @@ from __future__ import annotations
 
 import hashlib
 import json
+import logging
 import sqlite3
 import threading
 import time
@@ -21,14 +22,13 @@ from dataclasses import dataclass, field
 from enum import Enum
 from typing import Any, Callable
 
-import logging
-
 from superlocalmemory.storage.database import DatabaseManager
 
 logger = logging.getLogger("superlocalmemory.ingestion_command")
 
 _MATERIALIZATION_LOCKS = tuple(threading.RLock() for _ in range(64))
 _MAX_AUTOMATIC_MATERIALIZATION_ATTEMPTS = 10
+_NEVER_RETRY_AT = 9_999_999_999.0
 
 
 def _materialization_lock(operation_id: str) -> threading.RLock:
@@ -48,6 +48,10 @@ class IdempotencyConflict(ValueError):
     """The same idempotency key was reused for different immutable evidence."""
 
 
+class IngestionRejectedError(RuntimeError):
+    """Deterministic evidence policy produced no queryable projection."""
+
+
 class InvalidStateTransition(RuntimeError):
     """An ingestion operation attempted an illegal or stale transition."""
 
@@ -481,13 +485,14 @@ class IngestionOperationRepository:
         if target not in {IngestionState.COMPLETE, IngestionState.FAILED}:
             raise InvalidStateTransition(f"enriching -> {target.value}")
         current = self.get(operation_id)
-        # Attempt count AFTER this transition = current + 1 (the UPDATE adds
-        # the delta via attempt_count+delta but here we compare the pre-update
-        # value + 1 to the cap).
-        next_attempt_count = current.attempt_count + 1
+        # claim_enriching() already increments attempt_count before the
+        # materializer runs.  finish_enriching() records that claimed attempt;
+        # incrementing again here would dead-letter after only nine real tries
+        # while claiming that ten had run.
+        attempt_count = current.attempt_count
         is_exhausted = (
             target is IngestionState.FAILED
-            and next_attempt_count >= _MAX_AUTOMATIC_MATERIALIZATION_ATTEMPTS
+            and attempt_count >= _MAX_AUTOMATIC_MATERIALIZATION_ATTEMPTS
         )
         # Fix E: exhausted → far-future retry_at so list_materializable's
         # `next_retry_at <= now` clause never matches, excluding the
@@ -537,18 +542,19 @@ class IngestionOperationRepository:
                         "INSERT INTO dead_letter_operations "
                         "(original_op_id, operation_type, content, "
                         " metadata_json, error, attempt_count, "
-                        " first_attempt_at, profile_id) "
+                        " first_attempt_at, dead_lettered_at, profile_id) "
                         "VALUES (?, 'M018', ?, ?, ?, ?, "
-                        " (SELECT unixepoch(created_at) FROM ingestion_operations "
-                        "  WHERE operation_id=?), ?)",
+                        " (SELECT CAST(strftime('%s', created_at) AS REAL) "
+                        "  FROM ingestion_operations WHERE operation_id=?), ?, ?)",
                         (
                             operation_id,
                             current.raw_content,
                             json.dumps(current.metadata, separators=(",", ":"))
                             if current.metadata else None,
                             last_error or current.last_error,
-                            next_attempt_count,
+                            attempt_count,
                             operation_id,
+                            time.time(),
                             current.profile_id,
                         ),
                     )
@@ -556,7 +562,7 @@ class IngestionOperationRepository:
                         "Operation %s exhausted %d attempts — moved to dead-letter. "
                         "Last error: %s",
                         operation_id,
-                        next_attempt_count,
+                        attempt_count,
                         last_error or current.last_error,
                     )
         except InvalidStateTransition:
@@ -600,6 +606,107 @@ class IngestionOperationRepository:
                 ) from exc
         return self._from_row(rows[0])
 
+    def reap_stuck_enriching(
+        self,
+        *,
+        now: float | None = None,
+        limit: int = 100,
+    ) -> list[str]:
+        """Terminalize expired enrichment leases that exhausted automatic retries.
+
+        ``list_materializable`` intentionally excludes operations at the retry
+        cap.  If a worker dies while such an operation is still ``enriching``,
+        it otherwise becomes a permanent phantom: no worker can reclaim it and
+        it never reaches a terminal state.  Queryable facts are already durable,
+        so reaping abandons only optional derivation work.
+
+        Each candidate is transitioned with a compare-and-swap update in its
+        own bounded transaction.  A dead-letter record is supplemental; an
+        older database without M031 is still terminalized safely.
+        """
+        cutoff = time.time() if now is None else float(now)
+        batch_limit = max(1, min(int(limit), 500))
+        candidates = self.db.execute(
+            "SELECT operation_id, attempt_count, last_error, raw_content, "
+            "raw_metadata_json, profile_id FROM ingestion_operations "
+            "WHERE state='enriching' AND lease_expires_at <= ? "
+            "AND attempt_count >= ? ORDER BY updated_at, operation_id LIMIT ?",
+            (
+                cutoff,
+                _MAX_AUTOMATIC_MATERIALIZATION_ATTEMPTS,
+                batch_limit,
+            ),
+        )
+        reaped: list[str] = []
+        for row in candidates:
+            data = dict(row)
+            operation_id = str(data["operation_id"])
+            terminal_error = (
+                data["last_error"]
+                or "reaped: enrichment exhausted automatic attempts"
+            )
+            try:
+                with self.db.transaction():
+                    updated = self.db.execute(
+                        "UPDATE ingestion_operations SET state='failed', "
+                        "lease_owner='', lease_expires_at=0, next_retry_at=?, "
+                        "last_error=?, "
+                        "updated_at=strftime('%Y-%m-%dT%H:%M:%fZ', 'now') "
+                        "WHERE operation_id=? AND state='enriching' "
+                        "AND lease_expires_at <= ? AND attempt_count >= ? "
+                        "RETURNING operation_id",
+                        (
+                            _NEVER_RETRY_AT,
+                            terminal_error,
+                            operation_id,
+                            cutoff,
+                            _MAX_AUTOMATIC_MATERIALIZATION_ATTEMPTS,
+                        ),
+                    )
+                    if not updated:
+                        continue
+                    try:
+                        self.db.execute(
+                            "INSERT INTO dead_letter_operations "
+                            "(original_op_id, operation_type, content, "
+                            "metadata_json, error, attempt_count, "
+                            "first_attempt_at, dead_lettered_at, profile_id) "
+                            "VALUES (?, 'M018', ?, ?, ?, ?, "
+                            "(SELECT CAST(strftime('%s', created_at) AS REAL) "
+                            "FROM ingestion_operations WHERE operation_id=?), ?, ?)",
+                            (
+                                operation_id,
+                                data["raw_content"],
+                                data["raw_metadata_json"] or None,
+                                terminal_error,
+                                int(data["attempt_count"]),
+                                operation_id,
+                                time.time(),
+                                data["profile_id"],
+                            ),
+                        )
+                    except sqlite3.OperationalError as exc:
+                        if "no such table" not in str(exc).lower():
+                            raise
+                        logger.info(
+                            "Dead-letter table unavailable while reaping %s; "
+                            "terminal transition preserved",
+                            operation_id,
+                        )
+            except Exception:
+                logger.exception(
+                    "Failed to reap exhausted ingestion operation %s",
+                    operation_id,
+                )
+                continue
+            reaped.append(operation_id)
+            logger.warning(
+                "Reaped exhausted ingestion operation %s at attempt %d",
+                operation_id,
+                int(data["attempt_count"]),
+            )
+        return reaped
+
 
 @dataclass(frozen=True, slots=True)
 class MaterializationResult:
@@ -611,6 +718,7 @@ class MaterializationResult:
 
 
 QueryableWriter = Callable[[IngestionRequest, str], list[str]]
+AdmissionValidator = Callable[[IngestionRequest], None]
 Materializer = Callable[
     [IngestionOperation],
     list[str] | tuple[str, ...] | MaterializationResult,
@@ -627,6 +735,7 @@ class IngestionCommand:
         *,
         write_queryable: QueryableWriter,
         materialize: Materializer,
+        validate_admission: AdmissionValidator | None = None,
         project: Projector | None = None,
         derivation_version: str = "v3.7-ingestion-1",
         lease_seconds: float = 900.0,
@@ -634,6 +743,7 @@ class IngestionCommand:
         self.repository = repository
         self._write_queryable = write_queryable
         self._materializer = materialize
+        self._validate_admission = validate_admission
         self._projector = project
         self._derivation_version = derivation_version
         self._lease_seconds = max(1.0, float(lease_seconds))
@@ -704,13 +814,18 @@ class IngestionCommand:
         self, request: IngestionRequest,
     ) -> tuple[IngestionOperation, bool]:
         """Submit once and report whether this call created the operation."""
+        # Trust/authentication and deterministic policy checks belong before
+        # the durable transaction.  They may reject, log, or consult policy,
+        # but must never extend SQLite's writer critical section.
+        if self._validate_admission is not None:
+            self._validate_admission(request)
         with self.repository.db.transaction():
             operation, created = self.repository.create_with_status(request)
             if operation.state is not IngestionState.RAW:
                 return operation, created
             fact_ids = tuple(self._write_queryable(request, operation.operation_id))
             if not fact_ids:
-                raise RuntimeError("ingestion produced no queryable facts")
+                raise IngestionRejectedError("ingestion produced no queryable facts")
             receipt = self.repository.transition(
                 operation.operation_id,
                 expected=IngestionState.RAW,
@@ -735,19 +850,16 @@ class IngestionCommand:
         if operation.state is IngestionState.COMPLETE:
             return operation
         # Fix E: guard against re-attempting exhausted (dead-lettered) operations.
-        # attempt_count is incremented by claim_enriching BEFORE this method sees
-        # the new value, so the cap check uses the CURRENT (pre-claim) count.
-        # After the (cap-1)th attempt is claimed, attempt_count reaches cap-1;
-        # finish_enriching sees current.attempt_count=cap-1, next_attempt_count=cap
-        # and inserts the dead-letter row.  Any subsequent materialize() call
-        # (attempt_count already at cap-1 in FAILED state) would re-claim and
-        # insert a second dead-letter row — this guard prevents that.
+        # claim_enriching increments the count before each real attempt, and
+        # finish_enriching dead-letters the failure whose count reaches the
+        # cap. A subsequent materialize() call sees count==cap and must not
+        # re-claim or insert another dead-letter row.
         # ``force=True`` is the operator escape hatch used by retry() — it bypasses
         # this guard so an admin can manually re-enqueue a dead-lettered operation.
         if (
             not force
             and operation.state is IngestionState.FAILED
-            and operation.attempt_count >= _MAX_AUTOMATIC_MATERIALIZATION_ATTEMPTS - 1
+            and operation.attempt_count >= _MAX_AUTOMATIC_MATERIALIZATION_ATTEMPTS
         ):
             # Already dead-lettered — return FAILED without re-claiming.
             return operation

package/src/superlocalmemory/core/mutations.py

@@ -42,6 +42,8 @@ def delete_fact_authorized(
     *,
     trusted_actor_id: str,
     source_agent_id: str,
+    canonical_runtime: Any | None = None,
+    idempotency_key: str | None = None,
 ) -> dict[str, Any]:
     """Authorize, delete one profile-owned fact, then emit post hooks."""
     profile_id, context = _context(
@@ -51,15 +53,23 @@ def delete_fact_authorized(
         trusted_actor_id=trusted_actor_id,
         source_agent_id=source_agent_id,
     )
-    rows = engine._db.execute(
-        "SELECT content FROM atomic_facts "
-        "WHERE fact_id = ? AND profile_id = ? LIMIT 1",
-        (fact_id, profile_id),
-    )
-    if not rows:
-        return {"ok": False, "error": f"Memory {fact_id} not found"}
-    content_preview = dict(rows[0]).get("content", "")[:80]
-    engine._db.delete_fact(fact_id, profile_id=profile_id)
+    if canonical_runtime is not None:
+        result = dict(canonical_runtime.delete_fact(
+            profile_id, fact_id, idempotency_key=idempotency_key,
+        ))
+        if not result.get("ok"):
+            return {"ok": False, "error": f"Memory {fact_id} not found"}
+        content_preview = str(result.get("content_preview", ""))
+    else:
+        rows = engine._db.execute(
+            "SELECT content FROM atomic_facts "
+            "WHERE fact_id = ? AND profile_id = ? LIMIT 1",
+            (fact_id, profile_id),
+        )
+        if not rows:
+            return {"ok": False, "error": f"Memory {fact_id} not found"}
+        content_preview = dict(rows[0]).get("content", "")[:80]
+        engine._db.delete_fact(fact_id, profile_id=profile_id)
     try:
         from superlocalmemory.core.backend_orchestrator import get_orchestrator
         orchestrator = get_orchestrator()
@@ -86,6 +96,8 @@ def update_fact_authorized(
     *,
     trusted_actor_id: str,
     source_agent_id: str,
+    canonical_runtime: Any | None = None,
+    idempotency_key: str | None = None,
 ) -> dict[str, Any]:
     """Authorize a fact update and refresh semantic and lexical indexes."""
     if not content or not content.strip():
@@ -120,7 +132,17 @@ def update_fact_authorized(
                 updates["fisher_variance"] = fisher_variance
         except Exception as exc:
             logger.warning("UPDATE embedding refresh failed: %s", exc)
-    engine._db.update_fact(fact_id, updates, profile_id=profile_id)
+    if canonical_runtime is not None:
+        result = dict(canonical_runtime.update_fact(
+            profile_id,
+            fact_id,
+            updates,
+            idempotency_key=idempotency_key,
+        ))
+        if not result.get("ok"):
+            return {"ok": False, "error": f"Memory {fact_id} not found"}
+    else:
+        engine._db.update_fact(fact_id, updates, profile_id=profile_id)
     try:
         from superlocalmemory.core.backend_orchestrator import get_orchestrator
         orchestrator = get_orchestrator()

package/src/superlocalmemory/core/recall_pipeline.py

@@ -14,6 +14,8 @@ from __future__ import annotations
 import hashlib
 import hmac
 import logging
+import sqlite3
+from pathlib import Path
 from typing import TYPE_CHECKING, Any
 
 if TYPE_CHECKING:
@@ -212,6 +214,70 @@ def _behavioral_entities(results: list[Any], limit: int = 20) -> list[str]:
 _RANKING_MODES: frozenset[str] = frozenset({"off", "v1", "v2", "v2-ensemble"})
 
 
+class _ReadOnlyLearningView:
+    """Minimal learning-model reader that cannot initialise or mutate a DB."""
+
+    def __init__(self, db_path: Path) -> None:
+        self._db_path = db_path.resolve()
+
+    def _connection(self) -> sqlite3.Connection:
+        connection = sqlite3.connect(
+            f"{self._db_path.as_uri()}?mode=ro",
+            uri=True,
+            timeout=0.25,
+        )
+        connection.row_factory = sqlite3.Row
+        connection.execute("PRAGMA query_only=ON")
+        connection.execute("PRAGMA busy_timeout=250")
+        return connection
+
+    def count_signals(self, profile_id: str) -> int:
+        connection = self._connection()
+        try:
+            row = connection.execute(
+                "SELECT COUNT(*) AS count FROM learning_signals "
+                "WHERE profile_id = ?",
+                (profile_id,),
+            ).fetchone()
+            return int(row["count"]) if row else 0
+        finally:
+            connection.close()
+
+    def count_feedback(self, profile_id: str) -> int:
+        """Count legacy feedback without running schema initialization."""
+        connection = self._connection()
+        try:
+            row = connection.execute(
+                "SELECT COUNT(*) AS count FROM learning_feedback "
+                "WHERE profile_id = ?",
+                (profile_id,),
+            ).fetchone()
+            return int(row["count"]) if row else 0
+        finally:
+            connection.close()
+
+    def load_active_model(self, profile_id: str) -> dict[str, Any] | None:
+        connection = self._connection()
+        try:
+            row = connection.execute(
+                "SELECT state_bytes, bytes_sha256, feature_names, trained_at, "
+                "model_version FROM learning_model_state "
+                "WHERE profile_id = ? AND is_active = 1 LIMIT 1",
+                (profile_id,),
+            ).fetchone()
+            if row is None:
+                return None
+            return {
+                "state_bytes": bytes(row["state_bytes"]),
+                "bytes_sha256": row["bytes_sha256"],
+                "feature_names": row["feature_names"],
+                "trained_at": row["trained_at"],
+                "model_version": row["model_version"],
+            }
+        finally:
+            connection.close()
+
+
 def _resolve_ranking_mode(env: "dict[str, str] | os._Environ[str]") -> str:
     """Map the ``SLM_RANKING`` env var to a canonical mode.
 
@@ -239,6 +305,7 @@ def apply_ranking(
     *,
     config: Any = None,
     pipeline_version: str = "v2-ensemble",
+    record_signals: bool = False,
 ) -> "RecallResponse":
     """Run the ranking pipeline at the requested version.
 
@@ -273,6 +340,7 @@ def apply_ranking(
     try:
         response = apply_v2_bandit_ensemble(
             response, query, profile_id, query_id,
+            record_signals=record_signals,
         )
     except Exception as exc:  # pragma: no cover — defensive
         logger.debug("apply_ranking ensemble step skipped: %s", exc)
@@ -297,14 +365,16 @@ def apply_adaptive_ranking(
     Phase 3 (200+): LightGBM ML-based reranking.
     """
     from superlocalmemory.infra.data_root import state_path
-    from superlocalmemory.learning.feedback import FeedbackCollector
-
     learning_db = state_path("learning.db")
     if not learning_db.exists():
         return response
 
-    collector = FeedbackCollector(learning_db)
-    signal_count = collector.get_feedback_count(pid)
+    try:
+        signal_count = _ReadOnlyLearningView(learning_db).count_feedback(pid)
+    except sqlite3.Error:
+        # A pre-learning database may not have this optional table yet.
+        # Recall remains a query and cannot create it on demand.
+        return response
 
     if signal_count < 50:
         return response  # Phase 1: no change
@@ -388,7 +458,6 @@ def apply_v2_adaptive_ranking(
         from pathlib import Path as _P
 
         from superlocalmemory.infra.data_root import state_path
-        from superlocalmemory.learning.database import LearningDatabase
         from superlocalmemory.learning.model_cache import load_active
         from superlocalmemory.learning.ranker import AdaptiveRanker
 
@@ -397,7 +466,7 @@ def apply_v2_adaptive_ranking(
         if not db_path.exists():
             return response
 
-        db = LearningDatabase(db_path)
+        db = _ReadOnlyLearningView(db_path)
         signal_count = db.count_signals(profile_id)
         active = load_active(db, profile_id)
 
@@ -487,6 +556,7 @@ def apply_v2_bandit_ensemble(
     query_id: str,
     *,
     learning_db_path: Any = None,
+    record_signals: bool = False,
 ) -> RecallResponse:
     """Apply contextual bandit + optional LGBM ensemble rerank. Safe on error."""
     import os as _os
@@ -521,12 +591,14 @@ def apply_v2_bandit_ensemble(
         entity_count = 0
         # Use query_context hints if available on the engine — cheap fallback.
         bandit = ContextualBandit(db_path, profile_id)
-        choice = bandit.choose(
-            {
-                "query_type": response.query_type,
-                "entity_count": entity_count,
-            },
-            query_id,
+        context = {
+            "query_type": response.query_type,
+            "entity_count": entity_count,
+        }
+        choice = (
+            bandit.choose(context, query_id)
+            if record_signals
+            else bandit.choose_readonly(context)
         )
 
         # --- 2. apply channel weights -------------------------------------
@@ -536,9 +608,8 @@ def apply_v2_bandit_ensemble(
         active_model = None
         signal_count = 0
         try:
-            from superlocalmemory.learning.database import LearningDatabase
             from superlocalmemory.learning.model_cache import load_active
-            db = LearningDatabase(db_path)
+            db = _ReadOnlyLearningView(db_path)
             signal_count = db.count_signals(profile_id)
             active_model = load_active(db, profile_id)
         except Exception as exc:
@@ -561,28 +632,32 @@ def apply_v2_bandit_ensemble(
             logger.debug("v2 bandit ensemble_rerank skipped: %s", exc)
             final_results = weighted
 
-        # --- 5. enqueue signals (non-blocking) ----------------------------
-        try:
-            top20 = final_results[:20]
-            candidates = tuple(
-                SignalCandidate(
-                    fact_id=r.fact.fact_id,
-                    channel_scores=dict(r.channel_scores or {}),
-                    cross_encoder_score=None,
-                    result_dict={"fact_id": r.fact.fact_id,
-                                 "score": r.score},
+        # Recall is a query.  Implicit learning signals are deliberately
+        # disabled on this path: even a non-blocking enqueue eventually writes
+        # canonical/learning state and turns dashboard polling into contention.
+        # An explicit feedback command owns durable learning signals instead.
+        if record_signals:
+            try:
+                top20 = final_results[:20]
+                candidates = tuple(
+                    SignalCandidate(
+                        fact_id=r.fact.fact_id,
+                        channel_scores=dict(r.channel_scores or {}),
+                        cross_encoder_score=None,
+                        result_dict={"fact_id": r.fact.fact_id,
+                                     "score": r.score},
+                    )
+                    for r in top20
                 )
-                for r in top20
-            )
-            enqueue(SignalBatch(
-                profile_id=profile_id,
-                query_id=query_id,
-                query_text=query,
-                candidates=candidates,
-                query_context=query_context,
-            ))
-        except Exception as exc:
-            logger.debug("v2 bandit signal enqueue skipped: %s", exc)
+                enqueue(SignalBatch(
+                    profile_id=profile_id,
+                    query_id=query_id,
+                    query_text=query,
+                    candidates=candidates,
+                    query_context=query_context,
+                ))
+            except Exception as exc:
+                logger.debug("v2 bandit signal enqueue skipped: %s", exc)
 
         return RecallResponse(
             query=response.query,
@@ -665,15 +740,6 @@ def run_recall(
     agent hot path skips the internal round and delegates refinement to the
     calling LLM. ``fast=False`` forces the internal agentic round.
     """
-    # Pre-operation hooks
-    hook_ctx = {
-        "operation": "recall",
-        "agent_id": agent_id,
-        "profile_id": profile_id,
-        "query_preview": query[:100],
-    }
-    hooks.run_pre("recall", hook_ctx)
-
     m = mode or config.mode
 
     # v3.8.2: resolve the client-driven-agentic default when a caller left
@@ -772,33 +838,6 @@ def run_recall(
                 logger.debug("Agentic sufficiency skipped: %s", exc)
 
     _mark("agentic")
-    # V3.2: Log access for recalled facts (Phase 1)
-    if access_log and response.results:
-        try:
-            fact_ids = [r.fact.fact_id for r in response.results]
-            # Recall exposure logging is a durable analytics contract (tested),
-            # so it stays synchronous — it is a single batched write, cheap
-            # relative to the deferred spreading-activation cache / last_seen.
-            access_log.store_access_batch(
-                fact_ids=fact_ids,
-                profile_id=profile_id,
-                access_type="recall",
-            )
-        except Exception as exc:
-            logger.debug("Access log batch store failed: %s", exc)
-
-    # Query telemetry is permitted, but result-derived entities are not fed
-    # back as preferences.  Retrieval exposure is not positive feedback.
-    try:
-        _get_behavioral_tracker(db).record_query(
-            query=query,
-            query_type=response.query_type,
-            entities=[],
-            profile_id=profile_id,
-        )
-    except Exception as exc:
-        logger.debug("Behavioral tracking: %s", exc)
-
     # S8-ARC-04 (v3.4.22): unified ranking entry point. Single env-var
     # (SLM_RANKING=off|v1|v2|v2-ensemble) controls the pipeline. Legacy
     # SLM_V2_PIPELINE_DISABLED + SLM_BANDIT_DISABLED still honoured for
@@ -810,7 +849,7 @@ def run_recall(
         mode = _resolve_ranking_mode(_os.environ)
         response = apply_ranking(
             response, query, profile_id, query_id,
-            config=config, pipeline_version=mode,
+            config=config, pipeline_version=mode, record_signals=False,
         )
     except Exception as exc:
         logger.debug("Ranking pipeline skipped: %s", exc)
@@ -820,11 +859,6 @@ def run_recall(
     # mutation here.  Those state transitions require a separately authenticated
     # positive/negative outcome; merely returning a result is an exposure.
 
-    # Post-operation hooks (audit, trust signal, learning)
-    hook_ctx["result_count"] = len(response.results)
-    hook_ctx["query_type"] = response.query_type
-    hooks.run_post("recall", hook_ctx)
-
     from superlocalmemory.core.score_contract import finalize_score_contract
     finalize_score_contract(response)