superlocalmemory 3.8.1 → 3.8.3

package/src/superlocalmemory/core/component_registry.py

@@ -0,0 +1,487 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under AGPL-3.0-or-later - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""Central component / dependency capability registry (v3.8.2).
+
+Single source of truth for "what SLM needs, and whether it is present" —
+consumed by the self-heal thread (``server.unified_daemon._self_heal``),
+``slm doctor``, the ``GET /api/v3/components`` route, and the dashboard
+System-Health panel.
+
+Design principles
+-----------------
+* **Cheap, side-effect-free probes.** Probing NEVER loads torch into this
+  process and NEVER downloads anything: it uses import-spec checks, a
+  HuggingFace-cache lookup, and a short Ollama HTTP ping. Repair (model
+  download / pip install) is the self-heal thread's job — see
+  ``core.component_healer`` and ``unified_daemon._self_heal`` Step 0/0.5.
+* **Honest, actionable reporting.** Every component carries a status plus,
+  when missing, a plain-language fix command and whether SLM can auto-fix
+  it (``auto_fixable``). A non-technical user reads the report; the daemon
+  acts on ``auto_fixable`` items on their behalf.
+* **Live repair visibility.** A tiny thread-safe transient overlay lets the
+  self-heal thread mark a component ``retrying`` while a download/install is
+  in flight, so the dashboard shows progress. Overlays are advisory and are
+  cleared as soon as a fresh probe confirms the component is present.
+* **Immutability.** :class:`Component` is a frozen dataclass; snapshots
+  return freshly-built lists/dicts and are never mutated in place.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import threading
+import time
+from dataclasses import dataclass, replace
+from pathlib import Path
+from typing import Any
+
+# --------------------------------------------------------------------------
+# Status + category vocabulary
+# --------------------------------------------------------------------------
+
+STATUS_OK = "ok"            # present and usable
+STATUS_MISSING = "missing"  # absent — recall/feature degraded until fixed
+STATUS_DEGRADED = "degraded"  # present but not fully functional / unverifiable
+STATUS_RETRYING = "retrying"  # self-heal is actively downloading/installing
+
+CATEGORY_REQUIRED = "required"        # core recall breaks without it
+CATEGORY_RECOMMENDED = "recommended"  # semantic quality degrades without it
+CATEGORY_OPTIONAL = "optional"        # only used by an opt-in feature
+
+# Model repo ids — kept in sync with cli.setup_wizard (single definition here
+# would be ideal, but importing the wizard pulls its CLI surface; these three
+# strings are stable and asserted equal by tests).
+_EMBED_MODEL = "nomic-ai/nomic-embed-text-v1.5"
+_RERANKER_MODEL = "cross-encoder/ms-marco-MiniLM-L-12-v2"
+_COMPRESSOR_MODEL = "microsoft/llmlingua-2-xlm-roberta-large-meetingbank"
+
+
+@dataclass(frozen=True)
+class Component:
+    """One capability SLM depends on, and its current health.
+
+    Frozen: probes build new instances; the transient overlay uses
+    :func:`dataclasses.replace` to derive a modified copy rather than
+    mutating shared state.
+    """
+
+    key: str
+    label: str
+    category: str
+    status: str
+    detail: str = ""
+    fix_cmd: str = ""
+    auto_fixable: bool = False
+    last_checked: float = 0.0
+
+    def as_dict(self) -> dict[str, Any]:
+        return {
+            "key": self.key,
+            "label": self.label,
+            "category": self.category,
+            "status": self.status,
+            "detail": self.detail,
+            "fix_cmd": self.fix_cmd,
+            "auto_fixable": self.auto_fixable,
+            "last_checked": self.last_checked,
+        }
+
+
+# --------------------------------------------------------------------------
+# Environment helpers
+# --------------------------------------------------------------------------
+
+def pip_is_user_writable() -> bool:
+    """True when a plain ``pip install`` into this interpreter is expected to
+    succeed — i.e. the interpreter is NOT PEP-668 externally-managed.
+
+    Mirrors the doctor's PEP-668 check. Used to decide whether SLM may
+    auto-``pip install`` a pure-python component (sqlite-vec) on the user's
+    behalf, or must instead hand them a pipx/uv fix command.
+    """
+    try:
+        import sysconfig
+
+        stdlib = sysconfig.get_path("stdlib")
+        if not stdlib:
+            return True
+        return not (Path(stdlib) / "EXTERNALLY-MANAGED").exists()
+    except Exception:
+        # Unknown → be conservative and do NOT auto-install.
+        return False
+
+
+def _module_present(module: str) -> bool:
+    """True if ``module`` can be imported, WITHOUT importing it.
+
+    ``find_spec`` only resolves the loader; it never executes the module, so
+    this stays cheap and never pulls torch/lancedb/etc. into the daemon.
+    """
+    try:
+        return importlib.util.find_spec(module) is not None
+    except Exception:
+        # A broken/partial install can raise inside find_spec — treat as absent.
+        return False
+
+
+def _hf_model_cached(repo_id: str) -> bool | None:
+    """Heuristic: is ``repo_id`` already in the local HuggingFace cache?
+
+    Returns True/False, or None when it cannot be determined (huggingface_hub
+    not installed). Checks for the model's ``config.json`` — present for every
+    HF model — without loading torch/sentence-transformers. This is a fast
+    presence heuristic; the authoritative load-test lives in the healer's real
+    download call, which no-ops when the model is genuinely present.
+    """
+    try:
+        from huggingface_hub import try_to_load_from_cache
+    except Exception:
+        return None
+    try:
+        hit = try_to_load_from_cache(repo_id, "config.json")
+    except Exception:
+        return None
+    # try_to_load_from_cache returns a filesystem path (str) only when the file
+    # is cached; None when unknown, or a _CACHED_NO_EXIST sentinel object when
+    # known-absent. Testing for str is version-proof across huggingface_hub
+    # releases (the sentinel's import location has moved between versions).
+    return isinstance(hit, str)
+
+
+def _embedding_is_remote(config: Any) -> bool:
+    """True when embeddings come from a remote endpoint (no local model needed).
+
+    Delegates to the wizard's canonical detector; conservative False on any
+    import/attribute error so a local embedder is still probed.
+    """
+    try:
+        from superlocalmemory.cli.setup_wizard import _embedding_is_remote as _r
+
+        return bool(_r(config))
+    except Exception:
+        return False
+
+
+# --------------------------------------------------------------------------
+# Individual probes — each returns a Component (never raises)
+# --------------------------------------------------------------------------
+
+def probe_python() -> Component:
+    import sys
+
+    v = sys.version_info
+    ok = v >= (3, 11)
+    return Component(
+        key="python",
+        label="Python runtime",
+        category=CATEGORY_REQUIRED,
+        status=STATUS_OK if ok else STATUS_MISSING,
+        detail=f"{v.major}.{v.minor}.{v.micro}" + ("" if ok else " (need >= 3.11)"),
+        fix_cmd="" if ok else "Install Python 3.11+ from https://python.org/downloads/",
+        auto_fixable=False,
+        last_checked=time.time(),
+    )
+
+
+def probe_search_deps() -> Component:
+    """sentence-transformers / torch / scikit-learn — semantic recall channel."""
+    missing = [m for m in ("sentence_transformers", "torch", "sklearn")
+               if not _module_present(m)]
+    ok = not missing
+    return Component(
+        key="search_deps",
+        label="Semantic search dependencies",
+        category=CATEGORY_RECOMMENDED,
+        status=STATUS_OK if ok else STATUS_MISSING,
+        detail="sentence-transformers, torch, scikit-learn"
+        if ok else "missing: " + ", ".join(missing),
+        fix_cmd="" if ok else "pip install 'superlocalmemory[search]'",
+        # NOT background-auto-installed: torch is ~2GB — a silent multi-GB
+        # download is a surprise the user must consent to. Fix command guides
+        # them; the daemon does not pull it unasked.
+        auto_fixable=False,
+        last_checked=time.time(),
+    )
+
+
+def _probe_hf_model(key: str, label: str, repo_id: str, *,
+                    category: str, auto_fixable: bool,
+                    fix_cmd: str) -> Component:
+    cached = _hf_model_cached(repo_id)
+    if cached is True:
+        status, detail, fix = STATUS_OK, repo_id, ""
+    elif cached is False:
+        status, detail, fix = STATUS_MISSING, f"{repo_id} not in local cache", fix_cmd
+    else:
+        # huggingface_hub unavailable → can't verify; report degraded, not a
+        # false "missing" alarm.
+        status, detail, fix = (
+            STATUS_DEGRADED, "cannot verify (huggingface_hub unavailable)", "",
+        )
+    # Downloading model weights only helps when the runtime that loads them
+    # (sentence-transformers/torch) is already present. If search deps are
+    # missing, the fix is to install those first — not to pull weights.
+    can_auto = (
+        auto_fixable
+        and status == STATUS_MISSING
+        and _module_present("sentence_transformers")
+    )
+    return Component(
+        key=key, label=label, category=category,
+        status=status, detail=detail, fix_cmd=fix,
+        auto_fixable=can_auto,
+        last_checked=time.time(),
+    )
+
+
+def probe_embedder_model(config: Any = None) -> Component:
+    if config is not None and _embedding_is_remote(config):
+        return Component(
+            key="embedder_model", label="Embedding model",
+            category=CATEGORY_REQUIRED, status=STATUS_OK,
+            detail="remote embedding endpoint (no local model required)",
+            last_checked=time.time(),
+        )
+    return _probe_hf_model(
+        "embedder_model", "Embedding model", _EMBED_MODEL,
+        category=CATEGORY_REQUIRED, auto_fixable=True,
+        fix_cmd="slm doctor --fix   (or: slm warmup)",
+    )
+
+
+def probe_reranker_model(config: Any = None) -> Component:
+    # Only auto-heal / flag the reranker when the user actually has it enabled.
+    enabled = True
+    try:
+        enabled = bool(getattr(config.retrieval, "use_cross_encoder", True)) \
+            if config is not None else True
+    except Exception:
+        enabled = True
+    comp = _probe_hf_model(
+        "reranker_model", "Reranker model", _RERANKER_MODEL,
+        category=CATEGORY_RECOMMENDED if enabled else CATEGORY_OPTIONAL,
+        auto_fixable=enabled,
+        fix_cmd="slm doctor --fix",
+    )
+    if not enabled and comp.status == STATUS_MISSING:
+        # Not enabled → absent is expected, not a problem.
+        return replace(comp, status=STATUS_OK,
+                       detail="disabled (retrieval.use_cross_encoder=false)",
+                       fix_cmd="", auto_fixable=False)
+    return comp
+
+
+def probe_compressor_model() -> Component:
+    """LLMLingua-2 prose compressor (~560MB).
+
+    NEVER auto-downloaded by self-heal: it is large and only used by opt-in
+    aggressive optimize compression, where it lazy-downloads on first real
+    use. Reported as optional so the dashboard shows it without alarm.
+    """
+    comp = _probe_hf_model(
+        "compressor_model", "Prose compression model", _COMPRESSOR_MODEL,
+        category=CATEGORY_OPTIONAL, auto_fixable=False,
+        fix_cmd="Downloads automatically on first use of aggressive compression",
+    )
+    if comp.status == STATUS_MISSING:
+        return replace(comp, detail=comp.detail + " (lazy — downloads on first use)")
+    return comp
+
+
+def probe_sqlite_vec() -> Component:
+    present = _module_present("sqlite_vec")
+    writable = pip_is_user_writable()
+    if present:
+        status, detail, fix, auto = STATUS_OK, "sqlite-vec installed", "", False
+    elif writable:
+        status, detail, fix, auto = (
+            STATUS_MISSING, "sqlite-vec not installed (vector index)",
+            "slm doctor --fix   (or: pip install sqlite-vec)", True,
+        )
+    else:
+        status, detail, fix, auto = (
+            STATUS_MISSING, "sqlite-vec not installed; interpreter externally managed",
+            "pipx inject superlocalmemory sqlite-vec   (or use a uv/venv install)",
+            False,
+        )
+    return Component(
+        key="sqlite_vec", label="Vector index (sqlite-vec)",
+        category=CATEGORY_RECOMMENDED, status=status, detail=detail,
+        fix_cmd=fix, auto_fixable=auto, last_checked=time.time(),
+    )
+
+
+def _probe_optional_pkg(key: str, label: str, module: str, pip_name: str,
+                        note: str) -> Component:
+    present = _module_present(module)
+    return Component(
+        key=key, label=label, category=CATEGORY_OPTIONAL,
+        status=STATUS_OK if present else STATUS_MISSING,
+        detail=f"{pip_name} installed" if present else f"{note}",
+        fix_cmd="" if present else f"pip install {pip_name}",
+        auto_fixable=False,  # opt-in scale/feature backends — staged consent.
+        last_checked=time.time(),
+    )
+
+
+def probe_lancedb() -> Component:
+    return _probe_optional_pkg(
+        "lancedb", "LanceDB scale backend", "lancedb", "lancedb",
+        "not installed (optional large-scale vector backend)",
+    )
+
+
+def probe_cozo() -> Component:
+    return _probe_optional_pkg(
+        "cozo", "CozoDB graph backend", "pycozo", "pycozo",
+        "not installed (optional large-scale graph backend)",
+    )
+
+
+def probe_llmlingua() -> Component:
+    return _probe_optional_pkg(
+        "llmlingua", "LLMLingua compressor lib", "llmlingua", "llmlingua",
+        "not installed (optional prose compression)",
+    )
+
+
+def probe_ollama(config: Any = None) -> Component:
+    """Ollama reachability — only meaningful in Mode B.
+
+    Never auto-fixed: installing the Ollama binary or pulling a model is a
+    surprise network/disk action, so it is always a manual fix command.
+    """
+    mode = ""
+    api_base = "http://localhost:11434"
+    model = ""
+    try:
+        if config is not None:
+            mode = getattr(getattr(config, "mode", None), "value", "") or ""
+            api_base = getattr(getattr(config, "llm", None), "api_base", api_base) or api_base
+            model = getattr(getattr(config, "llm", None), "model", "") or ""
+    except Exception:
+        pass
+    if mode != "b":
+        return Component(
+            key="ollama", label="Ollama (local LLM)",
+            category=CATEGORY_OPTIONAL, status=STATUS_OK,
+            detail="not in use (Mode B only)", last_checked=time.time(),
+        )
+    try:
+        import httpx
+
+        resp = httpx.get(f"{api_base}/api/tags", timeout=5.0)
+        if resp.status_code == 200:
+            names = [m.get("name", "").split(":")[0]
+                     for m in resp.json().get("models", [])]
+            if model and model.split(":")[0] not in names:
+                return Component(
+                    key="ollama", label="Ollama (local LLM)",
+                    category=CATEGORY_RECOMMENDED, status=STATUS_DEGRADED,
+                    detail=f"running but '{model}' not pulled",
+                    fix_cmd=f"ollama pull {model}", auto_fixable=False,
+                    last_checked=time.time(),
+                )
+            return Component(
+                key="ollama", label="Ollama (local LLM)",
+                category=CATEGORY_RECOMMENDED, status=STATUS_OK,
+                detail=f"running, {len(names)} models", last_checked=time.time(),
+            )
+        status_detail = f"HTTP {resp.status_code}"
+    except Exception:
+        status_detail = f"not reachable at {api_base}"
+    return Component(
+        key="ollama", label="Ollama (local LLM)",
+        category=CATEGORY_RECOMMENDED, status=STATUS_MISSING,
+        detail=status_detail,
+        fix_cmd="Install/start Ollama: https://ollama.com  then: brew services start ollama",
+        auto_fixable=False, last_checked=time.time(),
+    )
+
+
+# --------------------------------------------------------------------------
+# Transient overlay — live self-heal progress
+# --------------------------------------------------------------------------
+
+_transient_lock = threading.Lock()
+_transient: dict[str, tuple[str, str]] = {}  # key -> (status, detail)
+
+
+def mark_transient(key: str, status: str, detail: str = "") -> None:
+    """Mark a component's live repair state (e.g. RETRYING while downloading)."""
+    with _transient_lock:
+        _transient[key] = (status, detail)
+
+
+def clear_transient(key: str) -> None:
+    with _transient_lock:
+        _transient.pop(key, None)
+
+
+def _apply_transient(comp: Component) -> Component:
+    with _transient_lock:
+        override = _transient.get(comp.key)
+    if override is None:
+        return comp
+    status, detail = override
+    # A confirmed-present component supersedes a stale "retrying" marker.
+    if comp.status == STATUS_OK:
+        return comp
+    return replace(comp, status=status, detail=detail or comp.detail)
+
+
+# --------------------------------------------------------------------------
+# Aggregation
+# --------------------------------------------------------------------------
+
+def probe_all(config: Any = None) -> list[Component]:
+    """Probe every component once and return a fresh list (transient overlaid)."""
+    probes = [
+        probe_python(),
+        probe_search_deps(),
+        probe_embedder_model(config),
+        probe_reranker_model(config),
+        probe_compressor_model(),
+        probe_sqlite_vec(),
+        probe_llmlingua(),
+        probe_lancedb(),
+        probe_cozo(),
+        probe_ollama(config),
+    ]
+    return [_apply_transient(c) for c in probes]
+
+
+def auto_fixable_missing(config: Any = None) -> list[Component]:
+    """The subset SLM may repair without asking — drives self-heal + doctor --fix."""
+    return [c for c in probe_all(config)
+            if c.status == STATUS_MISSING and c.auto_fixable]
+
+
+def snapshot(config: Any = None) -> dict[str, Any]:
+    """Full registry snapshot for /api/v3/components, doctor --json, dashboard."""
+    comps = probe_all(config)
+    counts = {STATUS_OK: 0, STATUS_MISSING: 0,
+              STATUS_DEGRADED: 0, STATUS_RETRYING: 0}
+    for c in comps:
+        counts[c.status] = counts.get(c.status, 0) + 1
+    missing_needed = [c for c in comps
+                      if c.status == STATUS_MISSING
+                      and c.category in (CATEGORY_REQUIRED, CATEGORY_RECOMMENDED)]
+    healthy = not missing_needed and counts[STATUS_RETRYING] == 0
+    return {
+        "components": [c.as_dict() for c in comps],
+        "summary": {
+            "ok": counts[STATUS_OK],
+            "missing": counts[STATUS_MISSING],
+            "degraded": counts[STATUS_DEGRADED],
+            "retrying": counts[STATUS_RETRYING],
+            "auto_fixable_missing": sum(
+                1 for c in comps
+                if c.status == STATUS_MISSING and c.auto_fixable
+            ),
+            "healthy": healthy,
+        },
+        "generated_at": time.time(),
+    }

package/src/superlocalmemory/core/config.py

@@ -273,6 +273,27 @@ class RetrievalConfig:
     agentic_max_rounds: int = 3
     agentic_confidence_threshold: float = 0.3
 
+    # v3.8.2: Client-driven agentic (the recall spine's global flag).
+    # The agent hot path (CLI / MCP / plugins) is consumed by a frontier
+    # LLM (Claude Code, Copilot, Codex, …) that reformulates multi-hop /
+    # low-confidence queries far better than the local Ollama model. So the
+    # hot path DELEGATES the agentic reformulation loop to that calling LLM:
+    # it returns fast local retrieval (all six channels + reranker) plus the
+    # confidence signals (no_confident_match / answer_confidence / abstained),
+    # and never spends an internal LLM round unless a caller asks for it.
+    #   True  (default) → hot path skips the internal agentic round
+    #                     (equivalent to fast=True) and the client drives
+    #                     refinement. Consistent ~1.5-2s recall, no LLM tail.
+    #   False           → hot path always runs the internal agentic round
+    #                     when a query looks multi-hop / low-confidence
+    #                     (equivalent to fast=False) — for deployments with
+    #                     no smart client in front of SLM.
+    # This ONLY sets the default when a caller does not pass ``fast``
+    # explicitly. The dashboard human path (search-all → cluster summary)
+    # passes fast=False directly and always keeps internal synthesis.
+    # Env kill-switch: SLM_HOT_PATH_INTERNAL_AGENTIC=1 forces internal-on.
+    client_driven_agentic: bool = True
+
     # Spreading activation
     spreading_activation_decay: float = 0.7
     spreading_activation_threshold: float = 0.1

package/src/superlocalmemory/core/embeddings.py

@@ -652,9 +652,22 @@ class EmbeddingService:
                 release_embedding_lock()
 
     def _reset_idle_timer(self) -> None:
-        """Reset the configurable worker-idle timer."""
+        """Reset the configurable worker-idle timer.
+
+        F5 fix: opportunistically evict the worker mid-window when memory
+        pressure is detected (``_check_memory_pressure`` returns False).
+        This prevents the idle-timeout window from holding a worker alive
+        while system memory is exhausted.
+        """
         if self._idle_timer is not None:
             self._idle_timer.cancel()
+        if not self._check_memory_pressure():
+            # Pressure detected — kill worker immediately; do not schedule a
+            # new idle timer so no further embedding work is attempted until
+            # the next explicit request reloads the worker.
+            self._kill_worker()
+            self._idle_timer = None
+            return
         self._idle_timer = threading.Timer(
             _IDLE_TIMEOUT_SECONDS, self.unload,
         )

package/src/superlocalmemory/core/engine.py

@@ -369,6 +369,7 @@ class MemoryEngine:
             from superlocalmemory.core.maintenance_scheduler import MaintenanceScheduler
             self._maintenance_scheduler = MaintenanceScheduler(
                 self._db, self._config, self._profile_id,
+                embedder=self._embedder,  # v3.8.2: periodic NULL-embedding self-heal
             )
             self._maintenance_scheduler.start()
         except Exception as exc:
@@ -612,7 +613,7 @@ class MemoryEngine:
         mode: Mode | None = None, limit: int = CANONICAL_RECALL_LIMIT,
         agent_id: str = "unknown",
         session_id: str | None = None,
-        fast: bool = False,
+        fast: bool | None = None,
         *,
         include_global: bool | None = None,
         include_shared: bool | None = None,
@@ -627,10 +628,13 @@ class MemoryEngine:
         ``put_nowait`` and the actual ``pending_outcomes`` INSERT runs
         on a background worker.
 
-        ``fast=True`` is the latency-bounded path: it retains all six local
-        retrieval channels but skips remote agentic verification. Use
-        ``fast=False`` only when maximum multi-round quality matters more than
-        response time.
+        ``fast`` controls only the internal agentic verification round; all six
+        local retrieval channels + reranker run regardless. ``fast=None`` (the
+        default) resolves to the client-driven-agentic policy: the agent hot
+        path skips the internal round (equivalent to ``fast=True``) and lets the
+        calling LLM drive query refinement using the returned confidence signals.
+        Pass ``fast=False`` to force the internal agentic round when no smart
+        client sits in front of SLM. See ``recall_pipeline.resolve_hot_path_fast``.
 
         Multi-scope: ``include_global`` / ``include_shared`` control which
         scopes participate in retrieval. ``None`` (the default) means "use the

package/src/superlocalmemory/core/ingestion_command.py

@@ -21,8 +21,12 @@ from dataclasses import dataclass, field
 from enum import Enum
 from typing import Any, Callable
 
+import logging
+
 from superlocalmemory.storage.database import DatabaseManager
 
+logger = logging.getLogger("superlocalmemory.ingestion_command")
+
 _MATERIALIZATION_LOCKS = tuple(threading.RLock() for _ in range(64))
 _MAX_AUTOMATIC_MATERIALIZATION_ATTEMPTS = 10
 
@@ -545,18 +549,31 @@ class IngestionCommand:
         interval = min(30.0, max(0.1, self._lease_seconds / 3.0))
 
         def heartbeat() -> None:
-            while not stop.wait(interval):
-                try:
-                    renewed = self.repository.renew_enriching_lease(
-                        operation_id,
-                        owner=self._owner,
-                        lease_seconds=self._lease_seconds,
-                    )
-                except sqlite3.Error:
-                    continue
-                if not renewed:
-                    lost.set()
-                    return
+            # F9 fix: outer broad catch ensures lost.set() is always called
+            # when the heartbeat thread dies for any reason (not just sqlite3.Error).
+            # Without this, an AttributeError or unexpected exception would kill
+            # the thread silently, leaving lost=False while the lease has expired.
+            try:
+                while not stop.wait(interval):
+                    try:
+                        renewed = self.repository.renew_enriching_lease(
+                            operation_id,
+                            owner=self._owner,
+                            lease_seconds=self._lease_seconds,
+                        )
+                    except sqlite3.Error:
+                        continue
+                    if not renewed:
+                        lost.set()
+                        return
+            except Exception:
+                logger.warning(
+                    "heartbeat thread died unexpectedly for operation %s — "
+                    "signalling lease lost",
+                    operation_id,
+                    exc_info=True,
+                )
+                lost.set()
 
         thread = threading.Thread(
             target=heartbeat,
@@ -569,10 +586,13 @@ class IngestionCommand:
         finally:
             stop.set()
             thread.join(timeout=max(1.0, interval * 2))
-        if lost.is_set():
-            raise LeaseLost(
-                f"ingestion lease lost for operation {operation_id}"
-            )
+            # F7 fix: check inside finally so LeaseLost wins even when callback
+            # raised.  Without this, the original exception would propagate and
+            # the LeaseLost signal would be masked.
+            if lost.is_set():
+                raise LeaseLost(
+                    f"ingestion lease lost for operation {operation_id}"
+                )
         return result
 
     def submit(self, request: IngestionRequest) -> IngestionOperation: