superlocalmemory 4.0.8 → 4.0.10

package/src/superlocalmemory/storage/database.py

@@ -40,6 +40,7 @@ from superlocalmemory.storage.models import (
     TemporalEvent,
     TrustScore,
 )
+from superlocalmemory.storage.embedding_codec import decode_embedding, encode_embedding
 from superlocalmemory.storage.write_lock import get_write_lock
 
 logger = logging.getLogger(__name__)
@@ -555,7 +556,7 @@ class DatabaseManager:
                  fact.interval_start, fact.interval_end,
                  fact.confidence, fact.importance, fact.evidence_count, fact.access_count,
                  json.dumps(fact.source_turn_ids), fact.session_id,
-                 _jd(fact.embedding), _jd(fact.fisher_mean), _jd(fact.fisher_variance),
+                 encode_embedding(fact.embedding), _jd(fact.fisher_mean), _jd(fact.fisher_variance),
                  fact.lifecycle.value, _jd(fact.langevin_position),
                  fact.emotional_valence, fact.emotional_arousal,
                  fact.signal_type.value, fact.created_at, _scope, _shared),
@@ -591,7 +592,7 @@ class DatabaseManager:
             evidence_count=d["evidence_count"], access_count=d["access_count"],
             source_turn_ids=_jl(d.get("source_turn_ids_json")),
             session_id=d.get("session_id", ""),
-            embedding=_jl(d.get("embedding"), None),
+            embedding=decode_embedding(d.get("embedding"), fact_id=d.get("fact_id", "<unknown>")),
             fisher_mean=_jl(d.get("fisher_mean"), None),
             fisher_variance=_jl(d.get("fisher_variance"), None),
             lifecycle=MemoryLifecycle(d["lifecycle"]) if d.get("lifecycle") else MemoryLifecycle.ACTIVE,
@@ -645,7 +646,7 @@ class DatabaseManager:
                 fact.access_count,
                 json.dumps(fact.source_turn_ids),
                 fact.session_id,
-                _jd(fact.embedding),
+                encode_embedding(fact.embedding),
                 _jd(fact.fisher_mean),
                 _jd(fact.fisher_variance),
                 fact.lifecycle.value,
@@ -685,8 +686,12 @@ class DatabaseManager:
             include_shared=include_shared,
             prefix="f",
         )
+        # Pins are injected straight into an agent's context, which makes this
+        # the most consequential display path in the class: a withheld row here
+        # is not merely shown, it is asserted as background truth.
         rows = self.execute(
             f"SELECT f.* FROM atomic_facts f WHERE {where} AND f.pinned = 1 "
+            f"{self.visible_fact_clause('f')} "
             "AND NOT EXISTS ("
             "    SELECT 1 FROM fact_temporal_validity tv "
             "    WHERE tv.fact_id = f.fact_id "
@@ -732,6 +737,65 @@ class DatabaseManager:
             self._archive_col_present = True
         return present
 
+    def _has_quarantine_column(self) -> bool:
+        """Whether atomic_facts carries the ``quarantined`` column.
+
+        Same shape as ``_has_archive_status``: cached once True (a column never
+        disappears), re-checked while absent so a later schema pass is picked
+        up. ``storage.schema.create_all_tables`` adds the column at every engine
+        init, so on any store the daemon has opened this is True — the guard
+        exists for a bare DatabaseManager pointed at a store that engine init
+        never touched, where filtering on the column would raise instead of
+        returning results.
+        """
+        if getattr(self, "_quarantine_col_present", False):
+            return True
+        present = any(
+            dict(row).get("name") == "quarantined"
+            for row in self.execute("PRAGMA table_info(atomic_facts)")
+        )
+        if present:
+            self._quarantine_col_present = True
+        return present
+
+    def visible_fact_clause(
+        self, prefix: str = "", *, include_quarantined: bool = False,
+    ) -> str:
+        """AND-clause excluding rows no caller should be shown as a memory.
+
+        Two exclusions, one definition: soft-deleted (``archive_status``) and
+        withheld (``quarantined``). Both are presence-guarded, because each
+        column arrives with a migration and may be absent on a store the engine
+        has not opened.
+
+        WHY THIS EXISTS AS A FUNCTION. 4.0.10 first put the quarantine filter in
+        ``get_facts_by_ids`` alone, reasoning that every retrieval channel
+        re-authorises through it and the engine drops what it cannot hydrate.
+        That reasoning was correct and the conclusion was wrong: it covered the
+        RECALL pipeline, and ``search``, ``list_recent``, ``fetch``, the MCP
+        resources and the dashboard's own search are not the recall pipeline.
+        Measured on a copy of the author's store, ``search_facts_fts`` returned
+        20 withheld rows out of 50 and ``get_all_facts`` 66 out of 400 — the
+        exact defect the design was meant to prevent, in the paths the design
+        never looked at.
+
+        There is no single SQL choke point in this codebase; ``_scope_where`` is
+        spliced against six other tables and cannot carry a fact column. So the
+        honest form of "one place" is one CLAUSE with an enumerable set of call
+        sites, and a test that fails when a read path does not use it:
+        tests/test_storage/test_no_read_path_shows_a_withheld_row.py
+
+        ``include_quarantined=True`` is for repair, erasure and export — paths
+        that must reach a withheld row to act on it.
+        """
+        table = f"{prefix}." if prefix else ""
+        clause = ""
+        if self._has_archive_status():
+            clause += f" AND COALESCE({table}archive_status, 'live') != 'archived'"
+        if not include_quarantined and self._has_quarantine_column():
+            clause += f" AND COALESCE({table}quarantined, 0) = 0"
+        return clause
+
     def get_all_facts(
         self, profile_id: str, limit: int | None = None,
         *,
@@ -754,14 +818,10 @@ class DatabaseManager:
         # hard, env-tunable ceiling even when the caller passes limit=None.
         if limit is None:
             limit = _unbounded_facts_ceiling()
-        # Archived facts are not live; never surface them in direct reads.
-        archive_clause = (
-            " AND COALESCE(archive_status, 'live') != 'archived'"
-            if self._has_archive_status()
-            else ""
-        )
+        # Soft-deleted and withheld rows are not memories a caller may see.
         rows = self.execute(
-            f"SELECT * FROM atomic_facts WHERE {where}{archive_clause} "
+            f"SELECT * FROM atomic_facts WHERE {where}"
+            f"{self.visible_fact_clause()} "
             "ORDER BY created_at DESC LIMIT ?",
             (*params, int(limit)),
         )
@@ -789,14 +849,12 @@ class DatabaseManager:
             include_global=include_global,
             include_shared=include_shared,
         )
-        archive_clause = (
-            " AND COALESCE(archive_status, 'live') != 'archived'"
-            if self._has_archive_status()
-            else ""
-        )
+        # Crossing a profile boundary is the last place a withheld row should
+        # appear: it would be a model's non-answer presented to somebody else
+        # as one of this profile's shared memories.
         rows = self.execute(
             f"SELECT * FROM atomic_facts WHERE {where} AND profile_id != ?"
-            f"{archive_clause} ORDER BY created_at DESC",
+            f"{self.visible_fact_clause()} ORDER BY created_at DESC",
             (*params, profile_id),
         )
         return [self._row_to_fact(r) for r in rows]
@@ -873,7 +931,13 @@ class DatabaseManager:
             raise ValueError(f"Disallowed column(s): {bad_keys}")
         clean: dict[str, Any] = {}
         for k, v in updates.items():
-            if isinstance(v, (list, dict)):
+            if k == "embedding":
+                # Embeddings are stored in the canonical binary form. Falling
+                # through to json.dumps here would write a text row back into a
+                # converted store, one fact at a time, undoing the conversion
+                # wherever a fact is updated.
+                clean[k] = encode_embedding(v) if v is not None else None
+            elif isinstance(v, (list, dict)):
                 clean[k] = json.dumps(v)
             elif isinstance(v, (MemoryLifecycle, FactType, SignalType)):
                 clean[k] = v.value
@@ -946,14 +1010,21 @@ class DatabaseManager:
         include_global: bool = False,
         include_shared: bool = False,
     ) -> int:
-        """Total fact count for a profile."""
+        """Memories this profile has, as the owner would count them.
+
+        Counts what a caller can be shown, which is why it applies
+        ``visible_fact_clause``. It fed the dashboard's "All memories 5,093" and
+        was counting 1,195 withheld summaries and every soft-deleted row into
+        that figure -- a number the owner reads as "how much do I remember".
+        """
         where, params = _scope_where(
             profile_id,
             include_global=include_global,
             include_shared=include_shared,
         )
         rows = self.execute(
-            f"SELECT COUNT(*) AS c FROM atomic_facts WHERE {where}", (*params,),
+            f"SELECT COUNT(*) AS c FROM atomic_facts WHERE {where}"
+            f"{self.visible_fact_clause()}", (*params,),
         )
         return int(rows[0]["c"]) if rows else 0
 
@@ -1223,16 +1294,15 @@ class DatabaseManager:
             include_shared=include_shared,
             prefix="f",
         )
-        # Archived facts must not surface via full-text search either.
-        archive_clause = (
-            " AND COALESCE(f.archive_status, 'live') != 'archived'"
-            if self._has_archive_status()
-            else ""
-        )
+        # Full-text search is a display path: the dashboard search box, the
+        # `search` tool and `fetch` all land here, and none of them go through
+        # the recall engine. Before 4.0.10 put the clause here it returned 20
+        # withheld rows out of 50 on the author's store.
         rows = self.execute(
             f"""SELECT f.* FROM atomic_facts_fts AS fts
                JOIN atomic_facts AS f ON f.fact_id = fts.fact_id
-               WHERE fts.atomic_facts_fts MATCH ? AND {where}{archive_clause}
+               WHERE fts.atomic_facts_fts MATCH ? AND {where}
+                     {self.visible_fact_clause('f')}
                ORDER BY fts.rank LIMIT ?""",
             (match_expr, *params, limit),
         )
@@ -1263,12 +1333,22 @@ class DatabaseManager:
     # ------------------------------------------------------------------
 
     def get_fact(self, fact_id: str, profile_id: str | None = None) -> AtomicFact | None:
-        """Get a single fact by ID.
+        """Get a single row by ID, exactly as stored. NOT a display path.
 
         C4 defense-in-depth: when ``profile_id`` is provided the lookup is
         tenant-scoped so a fact_id from another profile cannot resolve. Left
         optional (fact_id is a random UUID sourced from already-scoped queries)
         to avoid destabilizing the core store/consolidation write path.
+
+        DELIBERATELY UNFILTERED, and this is load-bearing. It applies neither
+        ``archive_status`` nor ``quarantined`` because it is the primitive that
+        write paths, correction handling and the 4.0.10 repair use to read a row
+        they already hold the id of — including a withheld one, which they must
+        be able to see in order to act on it. ``visible_fact_clause`` is for the
+        paths that answer a question; this one answers "what is in that row".
+
+        A caller taking a fact_id from user input and rendering the result wants
+        ``get_facts_by_ids`` instead.
         """
         if profile_id is not None:
             rows = self.execute(
@@ -1285,8 +1365,36 @@ class DatabaseManager:
         self, fact_ids: list[str], profile_id: str,
         include_global: bool = False,
         include_shared: bool = False,
+        *,
+        include_quarantined: bool = False,
     ) -> list[AtomicFact]:
-        """Get multiple facts by their IDs, scoped to a profile."""
+        """Get multiple facts by their IDs, scoped to a profile.
+
+        THIS IS THE PLACE QUARANTINE IS ENFORCED, and the only one.
+
+        Every retrieval channel re-authorises its candidates through here
+        (``retrieval/scope_policy.py`` — "candidate generators may use caches,
+        approximate indexes, or graph stores that are not the authorization
+        source of truth"), and the engine hydrates the fused set from here too.
+        A fact this method does not return has no content to show, and
+        ``retrieval/engine.py`` drops it: ``if fact is None: continue``. So one
+        clause here covers bm25, semantic, temporal, entity, hopfield and
+        spreading activation, in normal and deep recall alike, whether or not
+        the forgetting filter is registered.
+
+        The alternatives were checked and rejected. ``_scope_where`` looks like
+        the natural home but is spliced against ``graph_edges``,
+        ``temporal_events``, ``memories``, ``bm25_tokens``,
+        ``fact_temporal_validity`` and ``correction_cases`` as well as
+        ``atomic_facts``, so a column reference there breaks eight call sites.
+        ``ForgettingFilter`` is optional (it no-ops when forgetting is
+        disabled) and excludes nothing in deep recall.
+
+        ``include_quarantined=True`` is for repair, export and erasure — paths
+        that must be able to see a withheld row in order to act on it. It is
+        keyword-only and greppable on purpose: every caller that opts in is
+        meant to be found in one search.
+        """
         if not fact_ids:
             return []
         where, params = _scope_where(
@@ -1294,15 +1402,12 @@ class DatabaseManager:
             include_global=include_global,
             include_shared=include_shared,
         )
-        archive_clause = (
-            " AND COALESCE(archive_status, 'live') != 'archived'"
-            if self._has_archive_status()
-            else ""
-        )
         placeholders = ",".join("?" for _ in fact_ids)
         rows = self.execute(
             f"SELECT * FROM atomic_facts WHERE fact_id IN ({placeholders}) "
-            f"AND {where}{archive_clause} ORDER BY created_at DESC",
+            f"AND {where}"
+            f"{self.visible_fact_clause(include_quarantined=include_quarantined)} "
+            "ORDER BY created_at DESC",
             (*fact_ids, *params),
         )
         return [self._row_to_fact(r) for r in rows]

package/src/superlocalmemory/storage/embedding_codec.py

@@ -0,0 +1,129 @@
+"""Encode and decode atomic_facts.embedding values.
+
+All read and write paths for atomic_facts.embedding must go through
+``encode_embedding`` and ``decode_embedding``.  Centralising the logic here
+means a future format change requires one edit, not one per reader.
+
+Storage format
+--------------
+New rows: 768 × float32, little-endian, stored as SQLite BLOB (3,072 bytes).
+Legacy rows: JSON TEXT produced by json.dumps(list[float]).
+
+The read path accepts both formats so a partial backfill is safe by
+construction: callers see ``list[float]`` regardless of storage format.
+
+Error contract
+--------------
+A value that is neither valid JSON nor a well-formed float32 buffer raises
+``ValueError`` with the fact_id in the message.  Returning ``None`` silently
+for a malformed value is forbidden: the caller cannot distinguish a legitimate
+absent embedding from a data-loss event.
+"""
+from __future__ import annotations
+
+import json
+import logging
+from typing import TYPE_CHECKING
+
+import numpy as np
+
+logger = logging.getLogger(__name__)
+
+if TYPE_CHECKING:
+    pass
+
+__all__ = [
+    "EMBEDDING_DIM",
+    "EMBEDDING_BYTES",
+    "encode_embedding",
+    "decode_embedding",
+]
+
+EMBEDDING_DIM: int = 768
+EMBEDDING_BYTES: int = EMBEDDING_DIM * 4  # float32 = 4 bytes
+
+
+def encode_embedding(vec: list[float] | None) -> bytes | None:
+    """Serialise a float list to a binary float32 BLOB for SQLite storage.
+
+    Parameters
+    ----------
+    vec:
+        A list of floats, or ``None``.  Production embeddings are always
+        768-dimensional; the backfill script asserts the dimension before
+        calling this function.
+
+    Returns
+    -------
+    bytes | None
+        Little-endian float32 buffer, or ``None`` if *vec* is ``None``.
+    """
+    if vec is None:
+        return None
+    return np.array(vec, dtype=np.float32).tobytes()
+
+
+def decode_embedding(
+    raw: bytes | str | None,
+    *,
+    fact_id: str = "<unknown>",
+) -> list[float] | None:
+    """Deserialise an embedding from either TEXT (JSON) or BLOB (binary float32).
+
+    Parameters
+    ----------
+    raw:
+        The raw value from ``atomic_facts.embedding``:
+        ``None`` or empty string → absent embedding (returns ``None``).
+        ``bytes`` → binary float32 BLOB path.
+        ``str`` → legacy JSON TEXT path.
+    fact_id:
+        Included in any ``ValueError`` message for fast triage.
+
+    Returns
+    -------
+    list[float] | None
+        768-element list of floats, or ``None`` when the embedding is absent.
+
+    Raises
+    ------
+    ValueError
+        For a non-null value that is neither valid JSON nor a well-formed
+        float32 buffer.  Never returns ``None`` for a malformed value.
+    """
+    if raw is None or raw == "":
+        return None
+
+    if isinstance(raw, (bytes, bytearray)):
+        if len(raw) % 4 != 0 or len(raw) == 0:
+            raise ValueError(
+                f"Corrupt embedding buffer for fact {fact_id!r}: "
+                f"{len(raw)} bytes is not a multiple of 4 (float32)"
+            )
+        if len(raw) != EMBEDDING_BYTES:
+            # A torn write that happens to land on a 4-byte boundary is
+            # indistinguishable from a short vector by length alone, and it was
+            # accepted silently at debug level: 767 of 768 values still looks
+            # like a valid embedding, and every similarity computed from it is
+            # quietly wrong. Smaller vectors ARE legitimate in tests, so this is
+            # a warning rather than a refusal, but it must be visible.
+            logger.warning(
+                "embedding for fact %s is %d bytes (%d floats), not the expected "
+                "%d (%d floats) — expected only for a test vector; on a real "
+                "store this is a truncated write",
+                fact_id, len(raw), len(raw) // 4, EMBEDDING_BYTES, EMBEDDING_DIM,
+            )
+        return np.frombuffer(raw, dtype=np.float32).tolist()
+
+    if isinstance(raw, str):
+        try:
+            return json.loads(raw)
+        except (json.JSONDecodeError, ValueError) as exc:
+            raise ValueError(
+                f"Corrupt JSON embedding for fact {fact_id!r}: {exc}"
+            ) from exc
+
+    raise ValueError(
+        f"Unexpected embedding type {type(raw).__name__!r} for fact {fact_id!r}; "
+        f"expected bytes or str"
+    )

package/src/superlocalmemory/storage/embedding_migrator.py

@@ -29,6 +29,8 @@ from typing import TYPE_CHECKING, Any
 
 import numpy as np
 
+from superlocalmemory.storage.embedding_codec import encode_embedding
+
 if TYPE_CHECKING:
     from superlocalmemory.core.config import SLMConfig
 
@@ -185,7 +187,7 @@ def _activate_staged_vectors(
                 updated = conn.execute(
                     "UPDATE atomic_facts SET embedding = ? "
                     "WHERE fact_id = ? AND profile_id = ?",
-                    (embedding_json, fact_id, profile_id),
+                    (encode_embedding(vector), fact_id, profile_id),
                 )
                 if updated.rowcount != 1:
                     raise RuntimeError(
@@ -606,7 +608,7 @@ def backfill_missing_embeddings(
                 logger.warning("backfill: null vector for fact %s — skipping.", fid[:16])
                 continue
             try:
-                embedding_json = json.dumps(vec)
+                embedding_blob = encode_embedding(vec)
                 # Metadata is not an independent record: it is the pointer to
                 # a sqlite-vec row. Creating it before the vector payload leaves
                 # semantic recall permanently blind while reporting success.
@@ -638,7 +640,7 @@ def backfill_missing_embeddings(
                     # this remains the supported JSON-only fallback path.
                     db.execute(
                         "UPDATE atomic_facts SET embedding = ? WHERE fact_id = ?",
-                        (embedding_json, fid),
+                        (embedding_blob, fid),
                     )
                 except Exception:
                     if projection_written and vector_store is not None:

package/src/superlocalmemory/storage/migration_runner.py

@@ -4,8 +4,6 @@
 
 """Forward-only additive migrations for SLM v3.4.22.
 
-LLD reference: ``.backup/active-brain/lld/LLD-07-schema-migrations-and-security-primitives.md``
-Section 4 (Migration Runner).
 
 Contract:
   - ``apply_all(learning_db, memory_db, *, dry_run=False) -> dict`` —
@@ -34,6 +32,7 @@ catalogue and the public orchestration functions.
 from __future__ import annotations
 
 import logging
+import os
 import sqlite3
 from pathlib import Path
 
@@ -160,6 +159,9 @@ from superlocalmemory.storage.migrations import (
 from superlocalmemory.storage.migrations import (
     M042_correction_case_ledger as _M042,
 )
+from superlocalmemory.storage.migrations import (
+    M043_quarantine_display_summaries as _M043,
+)
 from superlocalmemory.storage._schema_version import (
     SUPPORTED_SCHEMA_VERSION,
     SchemaVersionError,
@@ -177,6 +179,10 @@ from superlocalmemory.storage._migration_internals import (
     _migration_log_exists,
     _read_log,
 )
+from superlocalmemory.storage.backup import (
+    _gc_old_backups,
+    _pre_migration_backup,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -309,6 +315,14 @@ DEFERRED_MIGRATIONS: list[Migration] = [
     # Main-line M034 is renumbered in V4. It must remain deferred because its
     # backfill joins engine-bootstrapped memory_scenes and atomic_facts.
     Migration(name=_M039.NAME, db_target="memory", ddl=_M039.DDL),
+    # M043 withholds model-written summaries from the retrieval corpus and
+    # un-hides the memories they displaced. Deferred because it reads and
+    # writes atomic_facts + fact_retention, both bootstrapped at engine init —
+    # the same reason M011/M013/M015/M016 are deferred. apply_deferred takes a
+    # verified snapshot before the first migration it actually applies, so the
+    # store is recoverable.
+    Migration(name=_M043.NAME, db_target="memory", ddl=_M043.DDL,
+              dependencies=(_M011.NAME,)),
 ]
 
 
@@ -375,6 +389,61 @@ def _bootstrap_learning_schema(learning_db: Path, *, dry_run: bool) -> str | Non
     return None
 
 
+def _foreign_live_daemon(memory_db: Path) -> "int | None":
+    """Return the pid of another live daemon holding this data dir, or None.
+
+    Migrations are not fenced against concurrent writers. The realistic hazard
+    is an OLD daemon still running after an upgrade while a NEW one starts: its
+    WAL appends continue while DDL is applied, which can make a migration fail
+    non-deterministically. The snapshot itself stays consistent — the SQLite
+    backup API copies committed pages only — and a racing migration is recorded
+    as ``failed`` and is non-fatal, so this does not corrupt data.
+
+    This detects the condition and reports it. It deliberately does NOT refuse:
+    ``apply_all`` runs inside the daemon's own startup, so refusing whenever "a
+    daemon is running" would refuse on itself, and blocking on a lock here would
+    risk wedging startup — a worse outcome than a retryable failed step.
+    """
+    try:
+        pid_file = memory_db.parent / "daemon.pid"
+        if not pid_file.is_file():
+            return None
+        pid = int(pid_file.read_text().strip() or 0)
+        if pid <= 0 or pid == os.getpid():
+            return None
+        os.kill(pid, 0)          # signal 0 tests liveness without touching it
+        return pid
+    except (OSError, ValueError):
+        return None
+
+
+def _nothing_left_to_apply(learning_db: Path, memory_db: Path) -> bool:
+    """True when every migration is already recorded in its target database.
+
+    Used to decide whether a snapshot is worth taking. A snapshot is only
+    valuable when something is about to change; taking one on a start where
+    nothing changes copies the ALREADY-MIGRATED store and then prunes a
+    generation — so after two such starts the last copy of the original is gone,
+    and the safety net has quietly deleted the thing it exists to protect.
+
+    Errs toward False, which means "take the snapshot" — the safe direction.
+    """
+    try:
+        for migration in MIGRATIONS:
+            db_path = _db_for(migration.db_target, learning_db, memory_db)
+            if not db_path.exists():
+                return False
+            conn = _connect(db_path)
+            try:
+                if not _deferred_already_applied(conn, migration.name):
+                    return False
+            finally:
+                conn.close()
+    except Exception:  # noqa: BLE001 — any doubt means take the snapshot
+        return False
+    return True
+
+
 def apply_all(
     learning_db: Path,
     memory_db: Path,
@@ -401,6 +470,44 @@ def apply_all(
     failed: list[str] = []
     details: dict[str, str] = {}
 
+    # Take a consistent snapshot of both databases before any migration runs.
+    # The backup uses the SQLite backup API so in-flight WAL writers are
+    # never captured mid-transaction. InsufficientDiskSpaceError propagates
+    # to the caller — migration is intentionally aborted when disk is too
+    # tight to keep a recoverable copy.
+    # A snapshot is only worth taking when something is about to change. This
+    # runs on every engine construction, not just upgrades, so snapshotting
+    # unconditionally meant an ordinary start copied the already-migrated store
+    # and pruned a generation — two extra starts and the original was gone.
+    _pending = not _nothing_left_to_apply(learning_db, memory_db)
+    if not dry_run and not _pending:
+        details["_backup"] = "skipped: every migration already applied"
+
+    if not dry_run and _pending:
+        _other = _foreign_live_daemon(memory_db)
+        if _other is not None:
+            logger.warning(
+                "Another SuperLocalMemory daemon (pid %s) is still running and "
+                "writing to this data directory. Migrations are not fenced "
+                "against concurrent writers, so a step may fail and need a "
+                "retry. Your data is not at risk: the snapshot copies committed "
+                "pages only, and a failed step is recorded, never forced. Stop "
+                "the other daemon and restart if a step fails.",
+                _other,
+            )
+            details["_concurrent_daemon_pid"] = str(_other)
+
+        backup_dir = _pre_migration_backup(
+            learning_db, memory_db,
+            backups_root=memory_db.parent / "pre-migration-snapshots",
+        )
+        # _pre_migration_backup returns the snapshots root itself, so this is
+        # the directory to prune. Passing .parent pointed the collector at the
+        # data directory, where it matched nothing and pruned nothing — leaving
+        # every snapshot on disk for ever.
+        _gc_old_backups(backup_dir)
+        details["_backup"] = str(backup_dir)
+
     schema_error = _bootstrap_learning_schema(learning_db, dry_run=dry_run)
     if schema_error is not None:
         failed.append("learning_schema_bootstrap")
@@ -461,6 +568,28 @@ def apply_all(
     }
 
 
+def _deferred_already_applied(conn: sqlite3.Connection, name: str) -> bool:
+    """True when ``name`` is recorded as ``complete`` in this database's migration_log.
+
+    Used only to decide whether a snapshot is needed. On any error it returns
+    False, which errs toward taking a snapshot — the safe direction.
+
+    A row whose status is ``failed`` or ``in_progress`` is NOT considered applied:
+    the runner will retry those entries, and the store deserves a fresh snapshot
+    before any retry runs DDL against it.  Counting any row (regardless of status)
+    caused ``_nothing_left_to_apply`` to return True after a failed migration,
+    so the retry ran against the already-partial store with no new safety copy.
+    """
+    try:
+        row = conn.execute(
+            "SELECT 1 FROM migration_log WHERE name = ? AND status = 'complete' LIMIT 1",
+            (name,),
+        ).fetchone()
+        return row is not None
+    except sqlite3.Error:
+        return False
+
+
 def apply_deferred(
     learning_db: Path,
     memory_db: Path,
@@ -495,6 +624,25 @@ def apply_deferred(
     failed: list[str] = []
     details: dict[str, str] = {}
 
+    # apply_all snapshots before it touches anything; this pass did not, yet it
+    # applies real DDL to both managed databases — including the column the
+    # daemon needs to start. An interrupted deferred pass therefore had no
+    # recoverable copy at all. The snapshot is taken LAZILY, immediately before
+    # the first migration that will actually be applied, so a pass with nothing
+    # to do costs no disk and does not capture post-init state unnecessarily.
+    _snapshot_state: dict[str, object] = {"taken": dry_run}
+
+    def _ensure_snapshot() -> None:
+        if _snapshot_state["taken"]:
+            return
+        _snapshot_state["taken"] = True
+        backup_dir = _pre_migration_backup(
+            learning_db, memory_db,
+            backups_root=memory_db.parent / "pre-migration-snapshots",
+        )
+        _gc_old_backups(backup_dir)
+        details["_deferred_backup"] = str(backup_dir)
+
     blocked: set[str] = set()
     for migration in DEFERRED_MIGRATIONS:
         unmet = [d for d in migration.dependencies if d in failed or d in blocked]
@@ -529,6 +677,9 @@ def apply_deferred(
                 )
                 continue
 
+            if not dry_run and not _deferred_already_applied(conn, migration.name):
+                _ensure_snapshot()
+
             outcome, detail = _apply_single(conn, migration, dry_run=dry_run)
             details[migration.name] = detail
             if outcome == "applied":