superlocalmemory 3.8.7 → 3.8.8

package/src/superlocalmemory/server/unified_daemon.py

@@ -1383,10 +1383,6 @@ async def lifespan(application: FastAPI):
                 application, new_config, mode_change=mode_change,
             )
         )
-        # v3.4.38: Wire module-level _engine for the pending materializer.
-        global _engine, _profile_runtime
-        _profile_runtime = profile_runtime
-        _engine = engine
         logger.info("Unified daemon: MemoryEngine initialized (mode=%s)", config.mode.value)
 
         # v3.5.0: Backend Orchestrator — CozoDB (graph) + LanceDB (vector) backends.
@@ -1631,8 +1627,13 @@ async def lifespan(application: FastAPI):
                     ]
                     if not with_emb:
                         continue
-                    if vs.count(pid) >= int(len(with_emb) * 0.98):
-                        continue  # already complete — no-op
+                    indexed_ids = vs.indexed_fact_ids(pid)
+                    missing = [
+                        item for item in with_emb
+                        if item[0] not in indexed_ids
+                    ]
+                    if not missing:
+                        continue  # every metadata pointer has a vec0 payload
                     # Fix: route each upsert through db._lock with cooperative
                     # yield between facts.  Previously called
                     # vs.rebuild_from_facts() which opens its own sqlite3
@@ -1653,8 +1654,8 @@ async def lifespan(application: FastAPI):
                     _pause = max(0.0, float(
                         _selfheal_os.environ.get("SLM_SELFHEAL_BATCH_PAUSE_S", "0.05")))
                     n = 0
-                    for _i in range(0, len(with_emb), _batch):
-                        _chunk = with_emb[_i:_i + _batch]
+                    for _i in range(0, len(missing), _batch):
+                        _chunk = missing[_i:_i + _batch]
                         with db._lock:
                             for _fact_id, _profile_id, _embedding in _chunk:
                                 try:
@@ -1670,8 +1671,8 @@ async def lifespan(application: FastAPI):
                         if _pause > 0:
                             _t.sleep(_pause)
                     logger.info(
-                        "VS backfill[%s]: indexed %d of %d embedded facts",
-                        pid, n, len(with_emb),
+                        "VS backfill[%s]: repaired %d of %d missing vectors",
+                        pid, n, len(missing),
                     )
             except Exception as exc:
                 logger.warning("Vector store backfill failed (non-fatal): %s", exc)
@@ -2132,6 +2133,15 @@ async def lifespan(application: FastAPI):
                     _mcp_lifespan_exc,
                 )
 
+        # Publish the resident engine to the pending materializer only after
+        # every synchronous startup writer has finished. Publishing it beside
+        # engine.initialize() allowed a stranded ingestion operation to race
+        # BackendOrchestrator status writes while the daemon was still inside
+        # lifespan, so /health could never become reachable.
+        global _engine, _profile_runtime
+        _profile_runtime = profile_runtime
+        _engine = engine
+
         # Uvicorn enters this lifespan only after it has bound the listener.
         # Publishing ``ready`` here prevents a failed competing process from
         # overwriting the live daemon descriptor before it owns the port.
@@ -3143,6 +3153,21 @@ def _register_daemon_routes(application: FastAPI) -> None:
             recall_health = {"recall_healthy": None}
         # Non-blocking peek: report status without forcing a re-init.
         engine = getattr(application.state, "engine", None)
+        embedding_ready = bool(_embedding_warm)
+        if engine is not None:
+            embedder = getattr(engine, "_embedder", None)
+            if embedder is None:
+                retrieval_engine = getattr(engine, "_retrieval_engine", None)
+                embedder = (
+                    getattr(retrieval_engine, "_embedder", None)
+                    if retrieval_engine is not None
+                    else None
+                )
+            if embedder is not None and hasattr(embedder, "is_warm"):
+                try:
+                    embedding_ready = bool(embedder.is_warm)
+                except Exception:
+                    embedding_ready = False
         migration_result = getattr(application.state, "migration_result", None)
         migration_failures = list(
             (migration_result or {}).get("failed", []) or []
@@ -3163,7 +3188,7 @@ def _register_daemon_routes(application: FastAPI) -> None:
                 writer_runtime is not None
                 and getattr(writer_runtime, "ready", False)
             ),
-            "embedding": bool(_embedding_warm),
+            "embedding": embedding_ready,
             "recall_health": recall_health.get("recall_healthy") is True,
             "migration_failures": migration_failures,
         }
@@ -3225,7 +3250,7 @@ def _register_daemon_routes(application: FastAPI) -> None:
             "version": getattr(application, 'version', 'unknown'),
             # v3.4.52: clients can poll this to wait for embedding model
             # readiness before issuing recall calls.
-            "embedding_warm": _embedding_warm,
+            "embedding_warm": embedding_ready,
             # v3.6.8: True iff the semantic channel actually fired on the last
             # health probe; includes self-heal counters.
             "recall_health": recall_health,
@@ -3976,6 +4001,10 @@ _materializer_stop = threading.Event()
 _materializer_thread: threading.Thread | None = None
 
 
+class _PendingProfileMismatchError(RuntimeError):
+    """A legacy pending row no longer matches the admitted profile lease."""
+
+
 def _materializer_actor_id() -> str:
     """Return the process-owned actor identity used by background writes."""
     descriptor = _ACTIVE_DAEMON_DESCRIPTOR
@@ -3986,7 +4015,13 @@ def _materializer_actor_id() -> str:
     return f"daemon-capability:{descriptor.capability_fingerprint}"
 
 
-def _run_materializer_operation(runtime, engine_supplier, operation):
+def _run_materializer_operation(
+    runtime,
+    engine_supplier,
+    operation,
+    *,
+    expected_profile_id: str | None = None,
+):
     """Run one bounded background unit against an admitted engine snapshot.
 
     Cooperative preemption: if a profile transition is already in progress,
@@ -3997,15 +4032,36 @@ def _run_materializer_operation(runtime, engine_supplier, operation):
     """
     # Writer-priority: don't acquire a new lease when a transition is draining.
     if runtime is not None and runtime.transitioning:
+        if expected_profile_id is not None:
+            raise _PendingProfileMismatchError(
+                "pending materialization deferred during profile transition"
+            )
         return None
-    with runtime.operation():
+    with runtime.operation() as snapshot:
+        if (
+            expected_profile_id is not None
+            and snapshot.profile_id != expected_profile_id
+        ):
+            raise _PendingProfileMismatchError(
+                "pending profile changed before materializer admission"
+            )
         # Resolve the engine only after admission. A concurrent mode/provider
         # reconfiguration may have replaced the module-level engine while this
         # worker was waiting at the transition barrier.
         engine = engine_supplier()
         if engine is None:
             return None
-        return operation(engine)
+        engine_profile_id = getattr(engine, "_profile_id", None)
+        if (
+            expected_profile_id is not None
+            and engine_profile_id != expected_profile_id
+        ):
+            raise _PendingProfileMismatchError(
+                "resident engine does not match pending profile"
+            )
+        from superlocalmemory.core.recall_gate import background_work
+        with background_work():
+            return operation(engine)
 
 
 def _materialize_ingestion_one_pass(
@@ -4021,6 +4077,19 @@ def _materialize_ingestion_one_pass(
     if _recalls_in_flight() > 0:
         return 0, 0
 
+    # A local sentence-transformers cold start can take minutes.  Remember's
+    # queryable projection is already durable, so defer enrichment until the
+    # daemon warmup/health monitor has proved the worker ready.  This preserves
+    # every enrichment layer while preventing a background cold load from
+    # monopolizing the same worker needed by foreground recall.
+    embedder = getattr(engine, "_embedder", None)
+    if embedder is not None and hasattr(embedder, "is_warm"):
+        try:
+            if not bool(embedder.is_warm):
+                return 0, 0
+        except Exception:
+            return 0, 0
+
     from superlocalmemory.core.engine_ingestion import build_engine_ingestion_command
     from superlocalmemory.core.ingestion_command import IngestionState
 
@@ -4081,6 +4150,12 @@ def _materialize_legacy_pending_item(engine, item: dict) -> str:
         IngestionState,
     )
 
+    expected_profile_id = str(item.get("profile_id") or "default")
+    if getattr(engine, "_profile_id", None) != expected_profile_id:
+        raise _PendingProfileMismatchError(
+            "legacy pending item does not match resident engine profile"
+        )
+
     metadata_value = item.get("metadata") or "{}"
     try:
         metadata = (
@@ -4101,7 +4176,7 @@ def _materialize_legacy_pending_item(engine, item: dict) -> str:
     command = build_engine_ingestion_command(engine)
     receipt = command.submit(IngestionRequest(
         content=item["content"],
-        profile_id=engine._profile_id,
+        profile_id=expected_profile_id,
         source_type=source_type,
         idempotency_key=idempotency_key,
         metadata=metadata,
@@ -4185,12 +4260,16 @@ def _start_pending_materializer() -> None:
                         time.sleep(0.5)
                         waits += 1
                     try:
+                        pending_profile_id = str(
+                            item.get("profile_id") or "default"
+                        )
                         operation_id = _run_materializer_operation(
                             runtime,
                             lambda: _ud._engine,
                             lambda admitted_engine: _materialize_legacy_pending_item(
                                 admitted_engine, item,
                             ),
+                            expected_profile_id=pending_profile_id,
                         )
                         if operation_id is None:
                             raise RuntimeError("resident engine became unavailable")
@@ -4205,6 +4284,15 @@ def _start_pending_materializer() -> None:
                             },
                             source_agent="materializer",
                         )
+                    except _PendingProfileMismatchError:
+                        # A profile transition committed after this row was
+                        # fetched but before it obtained an operation lease.
+                        # Leave it pending, without consuming a retry, so the
+                        # owning profile can safely drain it later.
+                        logger.debug(
+                            "Pending %d deferred after profile switch",
+                            item["id"],
+                        )
                     except Exception as exc:
                         logger.warning(
                             "Pending %d failed: %s", item["id"], exc,

package/src/superlocalmemory/storage/embedding_migrator.py

@@ -41,9 +41,7 @@ _BACKFILL_BATCH_SIZE = 50
 #: acquisition window.  Tunable via SLM_SELFHEAL_WRITE_DELAY_S; set to 0 to
 #: disable (only do this on single-user dev databases with no concurrency).
 #: Default 5 ms is imperceptible for humans but visible to the OS scheduler.
-_SELFHEAL_WRITE_DELAY_S: float = float(
-    _os.environ.get("SLM_SELFHEAL_WRITE_DELAY_S", "0.005")
-)
+_SELFHEAL_WRITE_DELAY_S: float = float(_os.environ.get("SLM_SELFHEAL_WRITE_DELAY_S", "0.005"))
 
 #: Max characters embedded per fact during backfill. The embedding model
 #: (nomic-embed-text-v1.5) truncates at ~8192 tokens anyway, but a raw
@@ -144,13 +142,15 @@ def check_embedding_migration(config: SLMConfig) -> bool:
         _write_stored_signature(config.base_dir, current_sig)
         logger.info(
             "Embedding signature normalized (no re-embed): %s ~= %s",
-            stored_sig, current_sig,
+            stored_sig,
+            current_sig,
         )
         return False
 
     logger.warning(
         "Embedding model changed: %s -> %s. Re-indexing required.",
-        stored_sig, current_sig,
+        stored_sig,
+        current_sig,
     )
     return True
 
@@ -176,8 +176,7 @@ def run_embedding_migration(
 
     # Get all fact IDs that need re-embedding (all facts for the profile).
     rows = db.execute(
-        "SELECT fact_id, content FROM atomic_facts "
-        "WHERE profile_id = ? ORDER BY created_at",
+        "SELECT fact_id, content FROM atomic_facts WHERE profile_id = ? ORDER BY created_at",
         (profile_id,),
     )
     facts = [(dict(r)["fact_id"], dict(r)["content"]) for r in rows]
@@ -189,7 +188,9 @@ def run_embedding_migration(
 
     logger.info(
         "Re-embedding %d facts with model %s (batch_size=%d)",
-        total, current_sig, _REINDEX_BATCH_SIZE,
+        total,
+        current_sig,
+        _REINDEX_BATCH_SIZE,
     )
 
     reindexed = 0
@@ -203,7 +204,9 @@ def run_embedding_migration(
         except Exception as exc:
             logger.error(
                 "Re-embedding batch %d-%d failed: %s. Stopping migration.",
-                i, i + len(batch), exc,
+                i,
+                i + len(batch),
+                exc,
             )
             break
 
@@ -219,22 +222,23 @@ def run_embedding_migration(
                 )
                 # Update embedding_metadata with new model name.
                 db.execute(
-                    "UPDATE embedding_metadata SET model_name = ? "
-                    "WHERE fact_id = ?",
+                    "UPDATE embedding_metadata SET model_name = ? WHERE fact_id = ?",
                     (config.embedding.model_name, fid),
                 )
                 reindexed += 1
             except Exception as exc:
                 logger.warning(
                     "Failed to update embedding for fact %s: %s",
-                    fid[:16], exc,
+                    fid[:16],
+                    exc,
                 )
 
     # Update stored signature after successful migration.
     _write_stored_signature(config.base_dir, current_sig)
     logger.info(
         "Embedding migration complete: %d/%d facts re-embedded.",
-        reindexed, total,
+        reindexed,
+        total,
     )
     return reindexed
 
@@ -243,6 +247,7 @@ def run_embedding_migration(
 # Backfill: embed facts that were NEVER embedded (embedding IS NULL)
 # ---------------------------------------------------------------------------
 
+
 def _count_null_embeddings(
     db: Any,
     profile_id: str,
@@ -255,8 +260,7 @@ def _count_null_embeddings(
         )
     else:
         rows = db.execute(
-            "SELECT count(*) AS c FROM atomic_facts "
-            "WHERE embedding IS NULL AND profile_id = ?",
+            "SELECT count(*) AS c FROM atomic_facts WHERE embedding IS NULL AND profile_id = ?",
             (profile_id,),
         )
     return int(rows[0]["c"]) if rows else 0
@@ -282,7 +286,7 @@ def backfill_missing_embeddings(
 
     Writes mirror :func:`run_embedding_migration` exactly:
     * ``atomic_facts.embedding`` ← ``json.dumps(vector)``
-    * ``embedding_metadata`` ← upserted row with current model name + dimension
+    * sqlite-vec + ``embedding_metadata`` ← one atomic projection pair
 
     Args:
         config: Active SLMConfig (provides profile_id, model name, dimension).
@@ -309,9 +313,7 @@ def backfill_missing_embeddings(
     profile_id = config.active_profile
 
     if embedder is None:
-        logger.warning(
-            "backfill_missing_embeddings: no embedder available — skipping."
-        )
+        logger.warning("backfill_missing_embeddings: no embedder available — skipping.")
         return {"scanned": 0, "embedded": 0, "remaining_null": 0}
 
     # ------------------------------------------------------------------
@@ -330,8 +332,7 @@ def backfill_missing_embeddings(
         )
 
     facts: list[tuple[str, str, str]] = [
-        (dict(r)["fact_id"], dict(r)["content"], dict(r)["profile_id"])
-        for r in rows
+        (dict(r)["fact_id"], dict(r)["content"], dict(r)["profile_id"]) for r in rows
     ]
     scanned = len(facts)
 
@@ -345,6 +346,24 @@ def backfill_missing_embeddings(
     current_model = config.embedding.model_name
     current_dim = config.embedding.dimension
     embedded = 0
+    vector_store = None
+    try:
+        from superlocalmemory.retrieval.vector_store import (
+            VectorStore,
+            VectorStoreConfig,
+        )
+
+        db_path = getattr(db, "db_path", None)
+        if db_path is not None:
+            vector_store = VectorStore(
+                db_path,
+                VectorStoreConfig(
+                    dimension=current_dim,
+                    model_name=current_model,
+                ),
+            )
+    except Exception as exc:
+        logger.debug("backfill: vector store unavailable: %s", exc)
 
     # ------------------------------------------------------------------
     # 2. Batch embed and write back
@@ -357,36 +376,39 @@ def backfill_missing_embeddings(
         fact_ids = [fid for fid, _, _ in batch]
         prof_ids = [pid for _, _, pid in batch]
 
-        # Attempt batch embed; fall back to per-fact on batch failure.
-        try:
-            vectors: list[Any] = embedder.embed_batch(texts)
-        except Exception as exc:
-            logger.warning(
-                "backfill: batch embed failed for facts %d-%d: %s — "
-                "retrying per-fact.",
-                batch_start,
-                batch_start + len(batch),
-                exc,
-            )
-            vectors = []
-            for text in texts:
-                try:
-                    vec = embedder.embed(text)
-                    vectors.append(vec)
-                except Exception as per_fact_exc:
-                    logger.warning(
-                        "backfill: per-fact embed failed for '%s...': %s",
-                        text[:40],
-                        per_fact_exc,
-                    )
-                    vectors.append(None)
+        # Attempt batch embed; fall back to per-fact on batch failure. Mark the
+        # whole inference burst as background work so the shared embedding
+        # service yields between items if a recall begins after the daemon's
+        # initial in-flight check.
+        from superlocalmemory.core.recall_gate import background_work
+
+        with background_work():
+            try:
+                vectors: list[Any] = embedder.embed_batch(texts)
+            except Exception as exc:
+                logger.warning(
+                    "backfill: batch embed failed for facts %d-%d: %s — retrying per-fact.",
+                    batch_start,
+                    batch_start + len(batch),
+                    exc,
+                )
+                vectors = []
+                for text in texts:
+                    try:
+                        vec = embedder.embed(text)
+                        vectors.append(vec)
+                    except Exception as per_fact_exc:
+                        logger.warning(
+                            "backfill: per-fact embed failed for '%s...': %s",
+                            text[:40],
+                            per_fact_exc,
+                        )
+                        vectors.append(None)
 
         # Write each successfully-embedded fact back to the DB.
         for fid, vec, pid in zip(fact_ids, vectors, prof_ids):
             if vec is None:
-                logger.warning(
-                    "backfill: null vector for fact %s — skipping.", fid[:16]
-                )
+                logger.warning("backfill: null vector for fact %s — skipping.", fid[:16])
                 continue
             try:
                 embedding_json = json.dumps(vec)
@@ -395,16 +417,24 @@ def backfill_missing_embeddings(
                     "UPDATE atomic_facts SET embedding = ? WHERE fact_id = ?",
                     (embedding_json, fid),
                 )
-                # Upsert embedding_metadata.  NULL-embedding facts have no row
-                # here yet, so we INSERT; if a row somehow exists, update it.
-                db.execute(
-                    "INSERT INTO embedding_metadata"
-                    "    (fact_id, profile_id, model_name, dimension)"
-                    " VALUES (?, ?, ?, ?)"
-                    " ON CONFLICT(fact_id) DO UPDATE SET"
-                    "    model_name = excluded.model_name",
-                    (fid, pid, current_model, current_dim),
-                )
+                # Metadata is not an independent record: it is the pointer to
+                # a sqlite-vec row. Creating it before the vector payload leaves
+                # semantic recall permanently blind while reporting success.
+                # VectorStore owns the atomic pair and repairs legacy orphans.
+                if (
+                    vector_store is not None
+                    and getattr(vector_store, "available", False)
+                    and not vector_store.upsert(
+                        fid,
+                        pid,
+                        vec,
+                        model_name=current_model,
+                    )
+                ):
+                    logger.warning(
+                        "backfill: vector projection failed for fact %s",
+                        fid[:16],
+                    )
                 embedded += 1
                 # Cooperative yield: release db._lock briefly so concurrent
                 # user writes can acquire it between facts.  Without this,
@@ -414,9 +444,7 @@ def backfill_missing_embeddings(
                 if _SELFHEAL_WRITE_DELAY_S > 0:
                     time.sleep(_SELFHEAL_WRITE_DELAY_S)
             except Exception as exc:
-                logger.warning(
-                    "backfill: failed to write fact %s: %s", fid[:16], exc
-                )
+                logger.warning("backfill: failed to write fact %s: %s", fid[:16], exc)
 
     # ------------------------------------------------------------------
     # 3. Count remaining NULLs (accounts for the limit; tells caller how