superlocalmemory 3.8.0 → 3.8.2

package/src/superlocalmemory/core/ingestion_command.py

@@ -21,9 +21,14 @@ from dataclasses import dataclass, field
 from enum import Enum
 from typing import Any, Callable
 
+import logging
+
 from superlocalmemory.storage.database import DatabaseManager
 
+logger = logging.getLogger("superlocalmemory.ingestion_command")
+
 _MATERIALIZATION_LOCKS = tuple(threading.RLock() for _ in range(64))
+_MAX_AUTOMATIC_MATERIALIZATION_ATTEMPTS = 10
 
 
 def _materialization_lock(operation_id: str) -> threading.RLock:
@@ -51,6 +56,10 @@ class OperationInProgress(RuntimeError):
     """Another live lease owner is materializing this operation."""
 
 
+class LeaseLost(OperationInProgress):
+    """The materializer no longer owns its durable operation lease."""
+
+
 def _canonical_json(value: Any) -> str:
     return json.dumps(value, sort_keys=True, separators=(",", ":"))
 
@@ -284,12 +293,19 @@ class IngestionOperationRepository:
             self._from_row(row)
             for row in self.db.execute(
                 "SELECT * FROM ingestion_operations "
-                "WHERE (state='queryable' AND created_at <= "
+                "WHERE attempt_count < ? AND ("
+                "(state='queryable' AND created_at <= "
                 "strftime('%Y-%m-%dT%H:%M:%fZ', 'now', ?)) "
                 "OR (state='failed' AND next_retry_at <= ?) "
-                "OR (state='enriching' AND lease_expires_at <= ?) "
+                "OR (state='enriching' AND lease_expires_at <= ?)) "
                 "ORDER BY created_at, rowid LIMIT ?",
-                (grace_modifier, now, now, max(1, int(limit))),
+                (
+                    _MAX_AUTOMATIC_MATERIALIZATION_ATTEMPTS,
+                    grace_modifier,
+                    now,
+                    now,
+                    max(1, int(limit)),
+                ),
             )
         ]
 
@@ -415,6 +431,27 @@ class IngestionOperationRepository:
         )
         return operation
 
+    def renew_enriching_lease(
+        self,
+        operation_id: str,
+        *,
+        owner: str,
+        lease_seconds: float,
+    ) -> bool:
+        """Extend a live lease only while the same owner still holds it."""
+        rows = self.db.execute(
+            "UPDATE ingestion_operations SET lease_expires_at=?, "
+            "updated_at=strftime('%Y-%m-%dT%H:%M:%fZ', 'now') "
+            "WHERE operation_id=? AND state='enriching' AND lease_owner=? "
+            "RETURNING operation_id",
+            (
+                time.time() + max(1.0, float(lease_seconds)),
+                operation_id,
+                owner,
+            ),
+        )
+        return bool(rows)
+
     def finish_enriching(
         self,
         operation_id: str,
@@ -469,6 +506,7 @@ class MaterializationResult:
 
     fact_ids: tuple[str, ...]
     derivation_state: dict[str, bool]
+    last_error: str = ""
 
 
 QueryableWriter = Callable[[IngestionRequest, str], list[str]]
@@ -500,6 +538,63 @@ class IngestionCommand:
         self._lease_seconds = max(1.0, float(lease_seconds))
         self._owner = f"ingestion-worker:{uuid.uuid4().hex}"
 
+    def _run_with_lease_heartbeat(
+        self,
+        operation_id: str,
+        callback: Callable[[], Any],
+    ) -> Any:
+        """Run slow work while periodically renewing its owner-bound lease."""
+        stop = threading.Event()
+        lost = threading.Event()
+        interval = min(30.0, max(0.1, self._lease_seconds / 3.0))
+
+        def heartbeat() -> None:
+            # F9 fix: outer broad catch ensures lost.set() is always called
+            # when the heartbeat thread dies for any reason (not just sqlite3.Error).
+            # Without this, an AttributeError or unexpected exception would kill
+            # the thread silently, leaving lost=False while the lease has expired.
+            try:
+                while not stop.wait(interval):
+                    try:
+                        renewed = self.repository.renew_enriching_lease(
+                            operation_id,
+                            owner=self._owner,
+                            lease_seconds=self._lease_seconds,
+                        )
+                    except sqlite3.Error:
+                        continue
+                    if not renewed:
+                        lost.set()
+                        return
+            except Exception:
+                logger.warning(
+                    "heartbeat thread died unexpectedly for operation %s — "
+                    "signalling lease lost",
+                    operation_id,
+                    exc_info=True,
+                )
+                lost.set()
+
+        thread = threading.Thread(
+            target=heartbeat,
+            name=f"ingestion-lease-heartbeat:{operation_id[:8]}",
+            daemon=True,
+        )
+        thread.start()
+        try:
+            result = callback()
+        finally:
+            stop.set()
+            thread.join(timeout=max(1.0, interval * 2))
+            # F7 fix: check inside finally so LeaseLost wins even when callback
+            # raised.  Without this, the original exception would propagate and
+            # the LeaseLost signal would be masked.
+            if lost.is_set():
+                raise LeaseLost(
+                    f"ingestion lease lost for operation {operation_id}"
+                )
+        return result
+
     def submit(self, request: IngestionRequest) -> IngestionOperation:
         operation, _created = self.submit_with_status(request)
         return operation
@@ -548,27 +643,41 @@ class IngestionCommand:
         )
         if enriching.state is IngestionState.COMPLETE:
             return enriching
-        if enriching.final_fact_ids:
+        if (
+            enriching.final_fact_ids
+            and all(enriching.derivation_state.values())
+        ):
             return self._project_and_complete(enriching)
         try:
+            # Materialization is a durable saga, not one long SQLite
+            # transaction. Extractors, embedders, and local model calls can
+            # take minutes on a mature installation; keeping a write
+            # transaction open across that work blocks every interactive
+            # remember/update/delete and makes the dashboard appear dead.
+            #
+            # The materializer commits its relational checkpoints in short
+            # database operations. The operation lease and derivation state
+            # remain the recovery boundary, and only the final state-machine
+            # checkpoint is grouped atomically below.
+            materialized = self._run_with_lease_heartbeat(
+                operation_id,
+                lambda: self._materializer(enriching),
+            )
+            if isinstance(materialized, MaterializationResult):
+                fact_ids = tuple(materialized.fact_ids)
+                derivation_state = dict(materialized.derivation_state)
+                materialization_error = materialized.last_error
+            else:
+                fact_ids = tuple(materialized)
+                derivation_state = {"materializer": True}
+                materialization_error = ""
+            if not fact_ids:
+                raise RuntimeError("materialization produced no final facts")
+            incomplete = sorted(
+                name for name, complete in derivation_state.items()
+                if not complete
+            )
             with self.repository.db.transaction():
-                materialized = self._materializer(enriching)
-                if isinstance(materialized, MaterializationResult):
-                    fact_ids = tuple(materialized.fact_ids)
-                    derivation_state = dict(materialized.derivation_state)
-                else:
-                    fact_ids = tuple(materialized)
-                    derivation_state = {"materializer": True}
-                if not fact_ids:
-                    raise RuntimeError("materialization produced no final facts")
-                incomplete = sorted(
-                    name for name, complete in derivation_state.items()
-                    if not complete
-                )
-                if incomplete:
-                    raise RuntimeError(
-                        "incomplete derivation stages: " + ", ".join(incomplete)
-                    )
                 checkpointed = self.repository.checkpoint_enriching(
                     operation_id,
                     final_fact_ids=fact_ids,
@@ -577,6 +686,21 @@ class IngestionCommand:
                     lease_owner=self._owner,
                     lease_seconds=self._lease_seconds,
                 )
+            if materialization_error or incomplete:
+                error = materialization_error or (
+                    "incomplete derivation stages: " + ", ".join(incomplete)
+                )
+                return self.repository.finish_enriching(
+                    operation_id,
+                    owner=self._owner,
+                    target=IngestionState.FAILED,
+                    final_fact_ids=fact_ids,
+                    derivation_version=self._derivation_version,
+                    derivation_state=derivation_state,
+                    last_error=error,
+                )
+        except LeaseLost:
+            raise
         except Exception as exc:
             return self.repository.finish_enriching(
                 operation_id,
@@ -598,10 +722,13 @@ class IngestionCommand:
                 owner=self._owner,
                 lease_seconds=self._lease_seconds,
             )
-            projection_state = (
-                dict(self._projector(operation))
-                if self._projector is not None
-                else {}
+            projection_state = self._run_with_lease_heartbeat(
+                operation.operation_id,
+                lambda: (
+                    dict(self._projector(operation))
+                    if self._projector is not None
+                    else {}
+                ),
             )
             combined = {**operation.derivation_state, **projection_state}
             incomplete = sorted(
@@ -619,6 +746,8 @@ class IngestionCommand:
                 derivation_version=self._derivation_version,
                 derivation_state=combined,
             )
+        except LeaseLost:
+            raise
         except Exception as exc:
             return self.repository.finish_enriching(
                 operation.operation_id,

package/src/superlocalmemory/core/injection.py

@@ -128,20 +128,25 @@ def _load_injection_config():
 
 
 def estimate_tokens(text: str) -> int:
-    """chars/4 heuristic. Optional tiktoken if installed (best-effort).
-
-    tiktoken is an optional dependency (pip install superlocalmemory[injection]).
-    Falls back to chars/4 if tiktoken is not installed or raises any error.
-    All exception types (MemoryError, SystemError etc.) are subclasses of
-    Exception in Python ≥ 3.11, so the bare except Exception covers all paths.
+    """Estimate context tokens without loading native code by default.
+
+    The stable default is the deterministic ``chars / 4`` heuristic.  Loading
+    the optional native ``tiktoken`` extension in a long-lived daemon that also
+    hosts torch/scipy workers can terminate the whole process on a native fault;
+    Python exception handling cannot recover from SIGBUS/SIGSEGV.  Users who
+    explicitly need exact OpenAI token counts can opt in with
+    ``SLM_INJECTION_EXACT_TOKENS=1``.
     """
     if not text:
         return 0
+    heuristic = max(1, len(text) // 4)
+    if os.environ.get("SLM_INJECTION_EXACT_TOKENS", "0") != "1":
+        return heuristic
     try:
         import tiktoken
         return len(tiktoken.get_encoding("cl100k_base").encode(text))
     except Exception:
-        return max(1, len(text) // 4)
+        return heuristic
 
 
 def resolve_budget(mode: str, cfg) -> int:

package/src/superlocalmemory/core/maintenance.py

@@ -109,6 +109,7 @@ def run_maintenance(
     db: DatabaseManager,
     config: SLMConfig,
     profile_id: str = "default",
+    embedder: object | None = None,
 ) -> dict[str, int]:
     """Run background maintenance on mathematical layers.
 
@@ -116,6 +117,11 @@ def run_maintenance(
         db: Database manager.
         config: Full SLM configuration.
         profile_id: Scope to this profile.
+        embedder: Optional embedder for self-healing NULL-embedding backfill.
+            When provided and NULL embeddings exist, up to 100 facts are
+            embedded per maintenance pass so the DB converges over time.
+            Pass ``None`` (default) to skip the backfill — existing callers
+            are unaffected.
 
     Returns:
         Dict of counts: langevin_updated, sheaf_checked, etc.
@@ -128,6 +134,7 @@ def run_maintenance(
         "entity_summaries_consolidated": 0,  # V3.4.40
         "orphan_metadata_gc": 0,             # v3.6.4 (P1-3)
         "expansion_backfilled": 0,           # T3b
+        "embeddings_backfilled": 0,          # v3.8.x NULL-embedding self-heal
     }
 
     # P1-3 (embeddings-vector-02): sweep orphaned embedding_metadata left by
@@ -138,6 +145,42 @@ def run_maintenance(
     except Exception as exc:  # pragma: no cover - defensive
         logger.debug("orphan metadata GC skipped: %s", exc)
 
+    # v3.8.x: self-healing NULL-embedding backfill.  Facts stored while the
+    # embedder was unavailable end up with NULL embedding and are invisible to
+    # semantic recall.  When an embedder is available, embed up to 100 facts
+    # per maintenance pass so the DB converges without blocking the caller.
+    if embedder is not None:
+        try:
+            from superlocalmemory.storage.embedding_migrator import (
+                backfill_missing_embeddings,
+            )
+
+            # Guard: skip entirely when nothing needs backfilling.
+            null_rows = db.execute(
+                "SELECT count(*) AS c FROM atomic_facts "
+                "WHERE embedding IS NULL AND profile_id = ?",
+                (profile_id,),
+            )
+            null_count = int(null_rows[0]["c"]) if null_rows else 0
+            if null_count > 0:
+                result = backfill_missing_embeddings(
+                    config,
+                    db,
+                    embedder,
+                    batch_size=50,
+                    limit=100,
+                )
+                counts["embeddings_backfilled"] = result["embedded"]
+                if result["embedded"] > 0:
+                    logger.info(
+                        "Maintenance embedding backfill: %d facts embedded, "
+                        "%d remaining.",
+                        result["embedded"],
+                        result["remaining_null"],
+                    )
+        except Exception as exc:
+            logger.debug("embedding backfill skipped during maintenance: %s", exc)
+
     facts = db.get_all_facts(profile_id)
     if not facts:
         return counts

package/src/superlocalmemory/core/maintenance_scheduler.py

@@ -8,7 +8,8 @@ V3.3.13: Periodically triggers Langevin/Ebbinghaus/Sheaf maintenance
 so users don't need to call run_maintenance manually.
 
 Configurable interval via ForgettingConfig.scheduler_interval_minutes.
-Defaults to 30 min. Disabled during benchmarks (no config.forgetting.enabled).
+Defaults to 30 min. Optional forgetting/math work follows
+``config.forgetting.enabled``; tier evaluation and bounded housekeeping do not.
 
 Part of Qualixar | Author: Varun Pratap Bhardwaj
 License: AGPL-3.0-or-later
@@ -39,10 +40,16 @@ class MaintenanceScheduler:
         db: DatabaseManager,
         config: SLMConfig,
         profile_id: str = "default",
+        embedder: object | None = None,
     ) -> None:
         self._db = db
         self._config = config
         self._profile_id = profile_id
+        # v3.8.2 self-heal: when provided, periodic maintenance backfills
+        # NULL embeddings so a DB stays fully queryable over time even if
+        # facts were stored while the embedder was unavailable. Runs
+        # independently of forgetting.enabled (see _run).
+        self._embedder = embedder
         self._timer: threading.Timer | None = None
         self._running = False
         self._interval = config.forgetting.scheduler_interval_minutes * 60.0
@@ -78,13 +85,44 @@ class MaintenanceScheduler:
         """Execute maintenance + auto-backup check, then schedule next run."""
         if not self._running:
             return
-        for profile_id in self._profile_ids():
+        # v3.8.2 self-heal: bounded NULL-embedding backfill runs every cycle
+        # INDEPENDENTLY of forgetting.enabled and across ALL profiles — a fact
+        # stored while the embedder was down must become queryable again without
+        # the user touching anything. Idempotent + bounded (200/pass) so it
+        # converges quietly and is a no-op once coverage is complete.
+        if self._embedder is not None:
             try:
-                from superlocalmemory.core.maintenance import run_maintenance
-                counts = run_maintenance(self._db, self._config, profile_id)
-                logger.info("Scheduled maintenance complete for %s: %s", profile_id, counts)
+                from superlocalmemory.storage.embedding_migrator import (
+                    backfill_missing_embeddings,
+                )
+                r = backfill_missing_embeddings(
+                    self._config, self._db, self._embedder,
+                    limit=50, all_profiles=True,
+                )
+                if r.get("embedded"):
+                    logger.info(
+                        "Self-heal backfill: %d embedded, %d remaining",
+                        r["embedded"], r["remaining_null"],
+                    )
             except Exception as exc:
-                logger.warning("Scheduled maintenance failed for %s: %s", profile_id, exc)
+                logger.debug("Self-heal backfill skipped: %s", exc)
+
+        for profile_id in self._profile_ids():
+            if self._config.forgetting.enabled:
+                try:
+                    from superlocalmemory.core.maintenance import run_maintenance
+                    counts = run_maintenance(self._db, self._config, profile_id)
+                    logger.info(
+                        "Scheduled maintenance complete for %s: %s",
+                        profile_id,
+                        counts,
+                    )
+                except Exception as exc:
+                    logger.warning(
+                        "Scheduled maintenance failed for %s: %s",
+                        profile_id,
+                        exc,
+                    )
 
             # V3.4.11: Graph pruning (remove orphan edges)
             try:

package/src/superlocalmemory/core/recall_pipeline.py

@@ -604,6 +604,33 @@ def apply_v2_bandit_ensemble(
 # run_recall  (was MemoryEngine.recall)
 # ---------------------------------------------------------------------------
 
+def resolve_hot_path_fast(fast: bool | None, config: "SLMConfig") -> bool:
+    """Resolve the recall ``fast`` flag when a caller leaves it unset (None).
+
+    v3.8.2 client-driven agentic: the agent hot path (CLI / MCP / plugins) is
+    consumed by a frontier LLM (Claude Code, Copilot, Codex, …) that reformulates
+    multi-hop / low-confidence queries far better than the local Ollama model.
+    So an unset ``fast`` defaults to True — skip the internal agentic round and
+    let the calling LLM drive refinement — whenever ``retrieval.client_driven_agentic``
+    is on (the ship default). An explicit ``True``/``False`` from the caller
+    always wins (the dashboard search path passes ``True`` for a snappy list;
+    a no-smart-client deployment can pass ``False``). Env override
+    ``SLM_HOT_PATH_INTERNAL_AGENTIC=1`` forces internal-agentic-on globally.
+
+    This is the single resolution point: every recall path (HTTP, MCP, CLI,
+    in-process adapter) funnels through ``run_recall`` and calls this, so the
+    client-driven default is consistent everywhere by construction.
+    """
+    if fast is not None:
+        return bool(fast)
+    import os
+    rc = getattr(config, "retrieval", None)
+    client_driven = bool(getattr(rc, "client_driven_agentic", True))
+    if os.environ.get("SLM_HOT_PATH_INTERNAL_AGENTIC") == "1":
+        client_driven = False
+    return client_driven
+
+
 def run_recall(
     query: str,
     profile_id: str,
@@ -620,7 +647,7 @@ def run_recall(
     hooks: HookRegistry,
     access_log: Any = None,
     auto_linker: Any = None,
-    fast: bool = False,
+    fast: bool | None = None,
     include_global: bool = False,
     include_shared: bool = False,
     window: str | tuple[str, str] | None = None,
@@ -632,8 +659,11 @@ def run_recall(
 
     Pipeline: retrieval -> agentic sufficiency (if configured) -> post-recall updates.
 
-    ``fast=True`` skips spreading activation and remote agentic verification,
-    retaining the bounded single-pass retrieval channels.
+    ``fast=True`` skips the internal agentic verification round while retaining
+    the six local retrieval channels + reranker. ``fast=None`` (unset) resolves
+    to the client-driven-agentic default (see ``resolve_hot_path_fast``): the
+    agent hot path skips the internal round and delegates refinement to the
+    calling LLM. ``fast=False`` forces the internal agentic round.
     """
     # Pre-operation hooks
     hook_ctx = {
@@ -646,6 +676,12 @@ def run_recall(
 
     m = mode or config.mode
 
+    # v3.8.2: resolve the client-driven-agentic default when a caller left
+    # ``fast`` unset (None). After this line ``fast`` is a concrete bool, so
+    # the agentic gate below (``if not fast``) behaves identically for every
+    # entry point that funnels through here.
+    fast = resolve_hot_path_fast(fast, config)
+
     # v3.5.0 diagnostic: per-stage recall timing under SLM_RECALL_TIMING=1.
     # Zero overhead when the env var is unset. Permanent observability hook.
     import os as _os_t
@@ -658,7 +694,9 @@ def run_recall(
             logger.warning("[RECALL-TIMING] %-22s %.0f ms",
                            _label, (_time_t.monotonic() - _t0) * 1000.0)
 
-    extra_disabled = {"spreading_activation"} if fast else None
+    # The interactive path must retain the complete local retrieval contract.
+    # Only agentic verification can invoke an unbounded model round.
+    extra_disabled = None
     response = retrieval_engine.recall(
         query, profile_id, m, limit,
         extra_disabled_channels=extra_disabled,