superlocalmemory 4.0.10 → 4.1.0

package/src/superlocalmemory/server/unified_daemon.py

@@ -1804,6 +1804,41 @@ def _stop_deployment_retention(application) -> bool:
     return True
 
 
+def _start_embedder_warmup(engine: object) -> "threading.Thread | None":
+    """Load the embedding model in the background. Never blocks startup.
+
+    Returns the thread so a test can join it; ``None`` when there is nothing to
+    warm. Every failure is "not warmed", never a failed startup: a daemon that
+    cannot embed still serves keyword recall and still stores memories, and the
+    materializer fills the vectors in afterwards either way.
+    """
+    embedder = getattr(engine, "_embedder", None)
+    if embedder is None or not hasattr(embedder, "embed"):
+        return None
+
+    from superlocalmemory.core.engine import _is_remote_embedder
+
+    if _is_remote_embedder(embedder):
+        # A hosted embedder has no model to load and warming it would spend a
+        # request, and money, on a sentence nobody asked about.
+        return None
+
+    def _warm() -> None:
+        started = time.time()
+        try:
+            embedder.embed("slm embedder warm-up")
+        except Exception as exc:  # pragma: no cover — warming is best effort
+            logger.debug("embedder warm-up failed (%s) — writes will defer", exc)
+            return
+        logger.info(
+            "Embedding model warm and ready (%.1fs)", time.time() - started,
+        )
+
+    thread = threading.Thread(target=_warm, daemon=True, name="slm-embed-warmup")
+    thread.start()
+    return thread
+
+
 @asynccontextmanager
 async def lifespan(application: FastAPI):
     """Initialize engine, workers, and optional services on startup."""
@@ -2088,6 +2123,40 @@ async def lifespan(application: FastAPI):
         engine = MemoryEngine(config)
         engine.initialize()
 
+        # Load the embedding model now, off the request path, the way the
+        # cross-encoder is already warmed at startup.
+        #
+        # A write embeds inline so the memory it stores can be found by asking a
+        # question rather than only by quoting its own words, and it gives that
+        # one second before deferring to the materializer. Loading the model
+        # takes 9.9-11.0 s here; once loaded an embed is 42 ms. So on a daemon
+        # that had not embedded yet, the first writes each waited the full second
+        # and stored no vector regardless -- and the model only ever loaded
+        # because some *recall* eventually paid for it. Whoever recalled first
+        # wore the cold start.
+        #
+        # This belongs to the daemon and not to engine wiring: the worker is a
+        # subprocess per engine, so warming from wiring would have every `slm
+        # status` spawn one and load a model it will never use.
+        _start_embedder_warmup(engine)
+
+        # Tell the hook subprocesses that skill evolution is on. The hook reads
+        # this env var as its fast-path signal and nothing ever set it, so the
+        # feature was off for everyone who had switched it on in config: the
+        # hook checked, found nothing, and returned False. Hooks are launched as
+        # children of this process and inherit its environment, which is the
+        # only channel between the two.
+        try:
+            if getattr(getattr(config, "evolution", None), "enabled", False):
+                os.environ["SLM_EVOLUTION_ENABLED"] = "1"
+            else:
+                # Cleared as well as set: a daemon restarted with the setting
+                # turned off must not leave the previous run's answer behind in
+                # an environment the next hook inherits.
+                os.environ.pop("SLM_EVOLUTION_ENABLED", None)
+        except Exception as _evo_exc:  # pragma: no cover — never block startup
+            logger.debug("evolution flag not exported: %s", _evo_exc)
+
         # Refresh migration state now that the engine is initialised.  Any
         # schema work the engine's own bootstrap may have applied (e.g.
         # runtime-table creation) is captured here so the dashboard and
@@ -2175,6 +2244,30 @@ async def lifespan(application: FastAPI):
                 "deferred migration runner crashed (non-fatal): %s", _dexc,
             )
 
+        # Move an existing store onto the graph and vector backends on the
+        # first start after an upgrade. They have shipped as required
+        # dependencies since 3.7 and sat unused, because building the
+        # projections was three manual commands almost nobody ran. Runs after
+        # the deferred migrations so it projects the converted store, and never
+        # fatal: if the libraries will not import or the projection does not
+        # match, the daemon serves from SQLite and says so.
+        try:
+            from superlocalmemory.core.scale_autopromote import (
+                auto_promote_scale_backends,
+            )
+            _promotion = auto_promote_scale_backends(config)
+            application.state.scale_autopromotion = _promotion.as_dict()
+            if _promotion.promoted and _promotion.restart_required:
+                logger.info(
+                    "graph and vector backends are promoted and serve after the "
+                    "next restart",
+                )
+        except Exception as _pexc:  # pragma: no cover — defensive
+            logger.warning("automatic backend promotion crashed (non-fatal): %s", _pexc)
+            application.state.scale_autopromotion = {
+                "attempted": True, "promoted": False, "reason": str(_pexc),
+            }
+
         # S9-DASH-02: start the outcome-queue worker so recall →
         # pending_outcomes is actually produced. Before v3.4.22 this
         # producer had zero callers and the closed-loop pipeline was
@@ -3041,6 +3134,18 @@ async def lifespan(application: FastAPI):
             await _cancel_fact_entity_association_repair(application)
             await _cancel_source_quality_repair(application)
 
+    # Stop the projection drain. Its queue is durable, so an interrupted pass
+    # costs a repeat of idempotent work and nothing else — but a live worker
+    # writing into RocksDB and Lance while the process tears down around it has
+    # no upside.
+    try:
+        from superlocalmemory.core.backend_orchestrator import get_orchestrator
+        _orch = get_orchestrator()
+        if _orch is not None:
+            _orch.stop()
+    except Exception as exc:  # pragma: no cover — defensive
+        logger.warning("projection drain shutdown failed: %s", exc)
+
     # Cancel the cross-platform sync loop (H-CONC-2) so adapter file I/O does
     # not outlive the daemon.
     try:
@@ -4072,12 +4177,26 @@ def _register_dashboard_routes(application: FastAPI) -> None:
         # 4.0.10: asset ?v= strings are now derived from file content instead of
         # being hand-written literals that tracked nothing. See
         # server/asset_versions.py — including what that does and does not fix.
-        from superlocalmemory.server.asset_versions import render_index
+        # Asset versioning is cosmetic. It must never be why this page 500s.
+        #
+        # The import is deferred (house style, keeps startup lean), which means
+        # it resolves at REQUEST time — so when `pip install -e .` replaced the
+        # installed package underneath a running daemon, this route began
+        # answering "Internal Server Error" on the dashboard while every other
+        # endpoint was fine. A stale hand-written version string is a trifle; a
+        # blank page is not. Fall back to the file as written.
+        try:
+            from superlocalmemory.server.asset_versions import render_index
 
-        return render_index(
-            index_path, UI_DIR,
-            substitutions={"__SLM_VERSION__": _SLM_VERSION},
-        )
+            return render_index(
+                index_path, UI_DIR, substitutions={"__SLM_VERSION__": _SLM_VERSION},
+            )
+        except Exception as exc:  # noqa: BLE001 — serve the page regardless
+            logger.warning(
+                "asset version rewrite unavailable, serving index.html as "
+                "written: %s: %s", type(exc).__name__, exc,
+            )
+            return index_path.read_text().replace("__SLM_VERSION__", _SLM_VERSION)
 
     @application.get("/favicon.ico", include_in_schema=False)
     async def favicon():
@@ -4177,6 +4296,20 @@ def _register_daemon_routes(application: FastAPI) -> None:
             "embedding": embedding_ready,
             "recall_health": recall_health.get("recall_healthy") is True,
             "migration_failures": migration_failures,
+            # WHY each one failed, not just which. The runner already produces
+            # a precise sentence per migration -- "safe repair did not restore
+            # M043_...", "schema verification failed ... : <sqlite error>" --
+            # and this endpoint computed it and then dropped it on the floor.
+            # A migration recorded ``complete`` in migration_log can still be
+            # reported failed here, because a completed migration is re-checked
+            # by its own verify() on every start; with only a name to go on,
+            # that reads as the health check contradicting the database. It is
+            # not: they are answering different questions, and this is the
+            # sentence that says which. Reported as #125.
+            "migration_failure_reasons": {
+                name: str(migration_details.get(name, "(no detail recorded)"))
+                for name in migration_failures
+            },
         }
         readiness["retrieval"] = bool(
             readiness["embedding"] and readiness["recall_health"]
@@ -4255,6 +4388,12 @@ def _register_daemon_routes(application: FastAPI) -> None:
             # reveal staleness on its own. Loopback-only, alongside the other
             # operational metadata.
             "version_integrity": _version_integrity_payload(),
+            # How far behind the second graph store is. A drain that stops
+            # advancing is the failure that does not announce itself: every
+            # other signal here stays green while the graph quietly diverges
+            # from the record, and the only symptom is answers that are subtly
+            # worse. A depth that does not fall is the thing to alert on.
+            "projection": _projection_health(),
         }
 
     @application.get("/recall")
@@ -4909,6 +5048,7 @@ def _register_daemon_routes(application: FastAPI) -> None:
         fact_count = 0
         entity_count = 0
         edge_count = 0
+        projection_queue_depth = 0
         if engine is not None:
             try:
                 fact_count = engine._db.get_fact_count(profile_snapshot.profile_id)
@@ -4925,6 +5065,11 @@ def _register_daemon_routes(application: FastAPI) -> None:
                 edge_count = int(dict(edges[0])["c"]) if edges else 0
             except Exception:
                 logger.debug("daemon status count query failed", exc_info=True)
+            try:
+                from superlocalmemory.storage import projection_outbox
+                projection_queue_depth = projection_outbox.depth(engine._db)
+            except Exception:
+                logger.debug("projection queue depth unavailable", exc_info=True)
         db_path = getattr(config, "db_path", None)
         db_size_mb = (
             round(db_path.stat().st_size / 1024 / 1024, 2)
@@ -4950,6 +5095,10 @@ def _register_daemon_routes(application: FastAPI) -> None:
             "legacy_port": _LEGACY_PORT,
             "profile": profile_snapshot.profile_id,
             "profile_generation": profile_snapshot.generation,
+            # Facts stored but not yet in the graph and vector projections. The
+            # CLI and MCP read their own status from here, so this is where the
+            # number has to be for all three surfaces to agree.
+            "projection_queue_depth": projection_queue_depth,
             # F2 fix: expose M028 backfill progress so operators can monitor
             # the post-upgrade fact/entity association repair state.
             "m028_backfill": getattr(
@@ -5561,6 +5710,38 @@ def _terminalize_orphan_operation(engine, operation_id: str) -> None:
         )
 
 
+def _projection_health() -> dict:
+    """Queue depth, stall count, and whether the worker is running.
+
+    Never raises: a health endpoint that fails because one of its fields could
+    not be computed is worse than the missing field.
+
+    Takes no application. It used to accept one and never read it -- the
+    orchestrator is a process singleton -- which made the signature claim a
+    dependency the body did not have, and made a test that handed it a broken
+    application look like it was exercising the failure path when it was only
+    observing whatever the process had already built. Compare
+    ``_ops_failure_counts`` directly below, which takes an application because it
+    genuinely reads one.
+    """
+    try:
+        from superlocalmemory.core.backend_orchestrator import get_orchestrator
+
+        orchestrator = get_orchestrator()
+        if orchestrator is None:
+            return {"available": False}
+        health = dict(orchestrator.outbox_health())
+        health["available"] = True
+        # One boolean an alert can key on without knowing what a healthy depth
+        # looks like on this store.
+        health["behind"] = bool(health.get("depth", 0)) or bool(
+            health.get("stalled", 0)
+        )
+        return health
+    except Exception as exc:  # noqa: BLE001
+        return {"available": False, "error": str(exc)[:120]}
+
+
 def _ops_failure_counts(engine, application) -> dict:
     """Return operational failure counts for /status and /health.
 

package/src/superlocalmemory/storage/_migration_internals.py

@@ -156,6 +156,16 @@ from superlocalmemory.storage.migrations import (
 from superlocalmemory.storage.migrations import (
     M043_quarantine_display_summaries as _M043,
 )
+from superlocalmemory.storage.migrations import (
+    M044_play_carries_its_own_evidence as _M044,
+)
+from superlocalmemory.storage.migrations import (
+    M045_fact_outcome_score as _M045,
+    M046_prospective_memory_has_its_own_name as _M046,
+    M047_fisher_vectors_are_stored_like_every_other_vector as _M047,
+    M048_upcoming_holds_only_what_is_upcoming as _M048,
+    M049_a_schema_version_marker_is_one_row as _M049,
+)
 
 # Emit under the runner's logger name so operational log filters that key on
 # "superlocalmemory.storage.migration_runner" keep matching after this split.
@@ -207,6 +217,12 @@ _MODULES = {
     _M041.NAME: _M041,
     _M042.NAME: _M042,
     _M043.NAME: _M043,
+    _M044.NAME: _M044,
+    _M045.NAME: _M045,
+    _M046.NAME: _M046,
+    _M047.NAME: _M047,
+    _M048.NAME: _M048,
+    _M049.NAME: _M049,
 }
 
 # Exact historical DDL fingerprints whose resulting schema is intentionally
@@ -245,9 +261,24 @@ def _ddl_hash(ddl: str) -> str:
     return hashlib.sha256(ddl.encode("utf-8")).hexdigest()
 
 
+#: How long a migration waits for a database another process is holding.
+#:
+#: Without this, SQLite raises SQLITE_BUSY the instant a write lock is taken,
+#: and a migration that runs while the daemon happens to be writing is recorded
+#: as failed rather than retried. A migration that rebuilds a table takes an
+#: exclusive lock, so it is exactly the one most likely to collide — and its
+#: failure leaves an upgrade wedged until someone notices.
+#:
+#: Fifteen seconds is longer than any single write this codebase performs and
+#: short enough that a genuinely stuck lock still surfaces as a failure rather
+#: than a hang.
+_MIGRATION_BUSY_TIMEOUT_MS = 15_000
+
+
 def _connect(db_path: Path) -> sqlite3.Connection:
     # isolation_level=None → we manage transactions explicitly via DDL.
     conn = sqlite3.connect(db_path, isolation_level=None)
+    conn.execute(f"PRAGMA busy_timeout = {_MIGRATION_BUSY_TIMEOUT_MS};")
     conn.execute("PRAGMA foreign_keys = OFF;")
     return conn
 

package/src/superlocalmemory/storage/_schema_version.py

@@ -20,10 +20,31 @@ from __future__ import annotations
 import sqlite3
 from pathlib import Path
 
-#: Highest schema_version this runner can write.  Matches the trailing serial
-#: of the latest migration (M042).  Increment when adding new migrations or
+#: Highest schema_version this runner can write.  Matches the trailing serial of
+#: the latest migration (M049).  Increment when adding new migrations or
 #: table-level breaking changes.
-SUPPORTED_SCHEMA_VERSION: int = 42
+#:
+#: This sat at 42 while M043, M044 and M045 shipped, so for three migrations the
+#: ceiling did not move and nothing prevented an older build from opening a
+#: newer store.  That was tolerable only because those three were additive: a
+#: build that did not know about a new column or table simply never read it.
+#:
+#: M046 is not additive.  It rebuilds ``atomic_facts`` with a constraint that
+#: rejects the value an older build classifies planned events as, so an older
+#: writer against a migrated store fails its INSERT.  The ceiling is what turns
+#: that from a lost memory into a refusal to start, which is why it moves here
+#: and why it moves to the trailing serial rather than to 43.
+#:
+#: M049 is additive — a unique index on ``schema_version`` plus the removal of
+#: the duplicate rows that index could not otherwise be created over — so an
+#: older build could in principle read a store it has touched.  The ceiling
+#: still moves, because the convention is "trailing serial, always": the cost of
+#: moving it for an additive migration is an older build declining a store it
+#: could have read, and the cost of NOT moving it for one that turns out not to
+#: be additive is a silent bad write.  Those are not comparable, and judging
+#: additivity per migration is exactly the judgement that let it fall three
+#: behind.
+SUPPORTED_SCHEMA_VERSION: int = 49
 
 
 class SchemaVersionError(RuntimeError):