superlocalmemory 4.0.8 → 4.0.9

package/src/superlocalmemory/storage/embedding_codec.py

@@ -0,0 +1,129 @@
+"""Encode and decode atomic_facts.embedding values.
+
+All read and write paths for atomic_facts.embedding must go through
+``encode_embedding`` and ``decode_embedding``.  Centralising the logic here
+means a future format change requires one edit, not one per reader.
+
+Storage format
+--------------
+New rows: 768 × float32, little-endian, stored as SQLite BLOB (3,072 bytes).
+Legacy rows: JSON TEXT produced by json.dumps(list[float]).
+
+The read path accepts both formats so a partial backfill is safe by
+construction: callers see ``list[float]`` regardless of storage format.
+
+Error contract
+--------------
+A value that is neither valid JSON nor a well-formed float32 buffer raises
+``ValueError`` with the fact_id in the message.  Returning ``None`` silently
+for a malformed value is forbidden: the caller cannot distinguish a legitimate
+absent embedding from a data-loss event.
+"""
+from __future__ import annotations
+
+import json
+import logging
+from typing import TYPE_CHECKING
+
+import numpy as np
+
+logger = logging.getLogger(__name__)
+
+if TYPE_CHECKING:
+    pass
+
+__all__ = [
+    "EMBEDDING_DIM",
+    "EMBEDDING_BYTES",
+    "encode_embedding",
+    "decode_embedding",
+]
+
+EMBEDDING_DIM: int = 768
+EMBEDDING_BYTES: int = EMBEDDING_DIM * 4  # float32 = 4 bytes
+
+
+def encode_embedding(vec: list[float] | None) -> bytes | None:
+    """Serialise a float list to a binary float32 BLOB for SQLite storage.
+
+    Parameters
+    ----------
+    vec:
+        A list of floats, or ``None``.  Production embeddings are always
+        768-dimensional; the backfill script asserts the dimension before
+        calling this function.
+
+    Returns
+    -------
+    bytes | None
+        Little-endian float32 buffer, or ``None`` if *vec* is ``None``.
+    """
+    if vec is None:
+        return None
+    return np.array(vec, dtype=np.float32).tobytes()
+
+
+def decode_embedding(
+    raw: bytes | str | None,
+    *,
+    fact_id: str = "<unknown>",
+) -> list[float] | None:
+    """Deserialise an embedding from either TEXT (JSON) or BLOB (binary float32).
+
+    Parameters
+    ----------
+    raw:
+        The raw value from ``atomic_facts.embedding``:
+        ``None`` or empty string → absent embedding (returns ``None``).
+        ``bytes`` → binary float32 BLOB path.
+        ``str`` → legacy JSON TEXT path.
+    fact_id:
+        Included in any ``ValueError`` message for fast triage.
+
+    Returns
+    -------
+    list[float] | None
+        768-element list of floats, or ``None`` when the embedding is absent.
+
+    Raises
+    ------
+    ValueError
+        For a non-null value that is neither valid JSON nor a well-formed
+        float32 buffer.  Never returns ``None`` for a malformed value.
+    """
+    if raw is None or raw == "":
+        return None
+
+    if isinstance(raw, (bytes, bytearray)):
+        if len(raw) % 4 != 0 or len(raw) == 0:
+            raise ValueError(
+                f"Corrupt embedding buffer for fact {fact_id!r}: "
+                f"{len(raw)} bytes is not a multiple of 4 (float32)"
+            )
+        if len(raw) != EMBEDDING_BYTES:
+            # A torn write that happens to land on a 4-byte boundary is
+            # indistinguishable from a short vector by length alone, and it was
+            # accepted silently at debug level: 767 of 768 values still looks
+            # like a valid embedding, and every similarity computed from it is
+            # quietly wrong. Smaller vectors ARE legitimate in tests, so this is
+            # a warning rather than a refusal, but it must be visible.
+            logger.warning(
+                "embedding for fact %s is %d bytes (%d floats), not the expected "
+                "%d (%d floats) — expected only for a test vector; on a real "
+                "store this is a truncated write",
+                fact_id, len(raw), len(raw) // 4, EMBEDDING_BYTES, EMBEDDING_DIM,
+            )
+        return np.frombuffer(raw, dtype=np.float32).tolist()
+
+    if isinstance(raw, str):
+        try:
+            return json.loads(raw)
+        except (json.JSONDecodeError, ValueError) as exc:
+            raise ValueError(
+                f"Corrupt JSON embedding for fact {fact_id!r}: {exc}"
+            ) from exc
+
+    raise ValueError(
+        f"Unexpected embedding type {type(raw).__name__!r} for fact {fact_id!r}; "
+        f"expected bytes or str"
+    )

package/src/superlocalmemory/storage/embedding_migrator.py

@@ -29,6 +29,8 @@ from typing import TYPE_CHECKING, Any
 
 import numpy as np
 
+from superlocalmemory.storage.embedding_codec import encode_embedding
+
 if TYPE_CHECKING:
     from superlocalmemory.core.config import SLMConfig
 
@@ -185,7 +187,7 @@ def _activate_staged_vectors(
                 updated = conn.execute(
                     "UPDATE atomic_facts SET embedding = ? "
                     "WHERE fact_id = ? AND profile_id = ?",
-                    (embedding_json, fact_id, profile_id),
+                    (encode_embedding(vector), fact_id, profile_id),
                 )
                 if updated.rowcount != 1:
                     raise RuntimeError(
@@ -606,7 +608,7 @@ def backfill_missing_embeddings(
                 logger.warning("backfill: null vector for fact %s — skipping.", fid[:16])
                 continue
             try:
-                embedding_json = json.dumps(vec)
+                embedding_blob = encode_embedding(vec)
                 # Metadata is not an independent record: it is the pointer to
                 # a sqlite-vec row. Creating it before the vector payload leaves
                 # semantic recall permanently blind while reporting success.
@@ -638,7 +640,7 @@ def backfill_missing_embeddings(
                     # this remains the supported JSON-only fallback path.
                     db.execute(
                         "UPDATE atomic_facts SET embedding = ? WHERE fact_id = ?",
-                        (embedding_json, fid),
+                        (embedding_blob, fid),
                     )
                 except Exception:
                     if projection_written and vector_store is not None:

package/src/superlocalmemory/storage/migration_runner.py

@@ -4,8 +4,6 @@
 
 """Forward-only additive migrations for SLM v3.4.22.
 
-LLD reference: ``.backup/active-brain/lld/LLD-07-schema-migrations-and-security-primitives.md``
-Section 4 (Migration Runner).
 
 Contract:
   - ``apply_all(learning_db, memory_db, *, dry_run=False) -> dict`` —
@@ -34,6 +32,7 @@ catalogue and the public orchestration functions.
 from __future__ import annotations
 
 import logging
+import os
 import sqlite3
 from pathlib import Path
 
@@ -177,6 +176,10 @@ from superlocalmemory.storage._migration_internals import (
     _migration_log_exists,
     _read_log,
 )
+from superlocalmemory.storage.backup import (
+    _gc_old_backups,
+    _pre_migration_backup,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -375,6 +378,61 @@ def _bootstrap_learning_schema(learning_db: Path, *, dry_run: bool) -> str | Non
     return None
 
 
+def _foreign_live_daemon(memory_db: Path) -> "int | None":
+    """Return the pid of another live daemon holding this data dir, or None.
+
+    Migrations are not fenced against concurrent writers. The realistic hazard
+    is an OLD daemon still running after an upgrade while a NEW one starts: its
+    WAL appends continue while DDL is applied, which can make a migration fail
+    non-deterministically. The snapshot itself stays consistent — the SQLite
+    backup API copies committed pages only — and a racing migration is recorded
+    as ``failed`` and is non-fatal, so this does not corrupt data.
+
+    This detects the condition and reports it. It deliberately does NOT refuse:
+    ``apply_all`` runs inside the daemon's own startup, so refusing whenever "a
+    daemon is running" would refuse on itself, and blocking on a lock here would
+    risk wedging startup — a worse outcome than a retryable failed step.
+    """
+    try:
+        pid_file = memory_db.parent / "daemon.pid"
+        if not pid_file.is_file():
+            return None
+        pid = int(pid_file.read_text().strip() or 0)
+        if pid <= 0 or pid == os.getpid():
+            return None
+        os.kill(pid, 0)          # signal 0 tests liveness without touching it
+        return pid
+    except (OSError, ValueError):
+        return None
+
+
+def _nothing_left_to_apply(learning_db: Path, memory_db: Path) -> bool:
+    """True when every migration is already recorded in its target database.
+
+    Used to decide whether a snapshot is worth taking. A snapshot is only
+    valuable when something is about to change; taking one on a start where
+    nothing changes copies the ALREADY-MIGRATED store and then prunes a
+    generation — so after two such starts the last copy of the original is gone,
+    and the safety net has quietly deleted the thing it exists to protect.
+
+    Errs toward False, which means "take the snapshot" — the safe direction.
+    """
+    try:
+        for migration in MIGRATIONS:
+            db_path = _db_for(migration.db_target, learning_db, memory_db)
+            if not db_path.exists():
+                return False
+            conn = _connect(db_path)
+            try:
+                if not _deferred_already_applied(conn, migration.name):
+                    return False
+            finally:
+                conn.close()
+    except Exception:  # noqa: BLE001 — any doubt means take the snapshot
+        return False
+    return True
+
+
 def apply_all(
     learning_db: Path,
     memory_db: Path,
@@ -401,6 +459,44 @@ def apply_all(
     failed: list[str] = []
     details: dict[str, str] = {}
 
+    # Take a consistent snapshot of both databases before any migration runs.
+    # The backup uses the SQLite backup API so in-flight WAL writers are
+    # never captured mid-transaction. InsufficientDiskSpaceError propagates
+    # to the caller — migration is intentionally aborted when disk is too
+    # tight to keep a recoverable copy.
+    # A snapshot is only worth taking when something is about to change. This
+    # runs on every engine construction, not just upgrades, so snapshotting
+    # unconditionally meant an ordinary start copied the already-migrated store
+    # and pruned a generation — two extra starts and the original was gone.
+    _pending = not _nothing_left_to_apply(learning_db, memory_db)
+    if not dry_run and not _pending:
+        details["_backup"] = "skipped: every migration already applied"
+
+    if not dry_run and _pending:
+        _other = _foreign_live_daemon(memory_db)
+        if _other is not None:
+            logger.warning(
+                "Another SuperLocalMemory daemon (pid %s) is still running and "
+                "writing to this data directory. Migrations are not fenced "
+                "against concurrent writers, so a step may fail and need a "
+                "retry. Your data is not at risk: the snapshot copies committed "
+                "pages only, and a failed step is recorded, never forced. Stop "
+                "the other daemon and restart if a step fails.",
+                _other,
+            )
+            details["_concurrent_daemon_pid"] = str(_other)
+
+        backup_dir = _pre_migration_backup(
+            learning_db, memory_db,
+            backups_root=memory_db.parent / "pre-migration-snapshots",
+        )
+        # _pre_migration_backup returns the snapshots root itself, so this is
+        # the directory to prune. Passing .parent pointed the collector at the
+        # data directory, where it matched nothing and pruned nothing — leaving
+        # every snapshot on disk for ever.
+        _gc_old_backups(backup_dir)
+        details["_backup"] = str(backup_dir)
+
     schema_error = _bootstrap_learning_schema(learning_db, dry_run=dry_run)
     if schema_error is not None:
         failed.append("learning_schema_bootstrap")
@@ -461,6 +557,28 @@ def apply_all(
     }
 
 
+def _deferred_already_applied(conn: sqlite3.Connection, name: str) -> bool:
+    """True when ``name`` is recorded as ``complete`` in this database's migration_log.
+
+    Used only to decide whether a snapshot is needed. On any error it returns
+    False, which errs toward taking a snapshot — the safe direction.
+
+    A row whose status is ``failed`` or ``in_progress`` is NOT considered applied:
+    the runner will retry those entries, and the store deserves a fresh snapshot
+    before any retry runs DDL against it.  Counting any row (regardless of status)
+    caused ``_nothing_left_to_apply`` to return True after a failed migration,
+    so the retry ran against the already-partial store with no new safety copy.
+    """
+    try:
+        row = conn.execute(
+            "SELECT 1 FROM migration_log WHERE name = ? AND status = 'complete' LIMIT 1",
+            (name,),
+        ).fetchone()
+        return row is not None
+    except sqlite3.Error:
+        return False
+
+
 def apply_deferred(
     learning_db: Path,
     memory_db: Path,
@@ -495,6 +613,25 @@ def apply_deferred(
     failed: list[str] = []
     details: dict[str, str] = {}
 
+    # apply_all snapshots before it touches anything; this pass did not, yet it
+    # applies real DDL to both managed databases — including the column the
+    # daemon needs to start. An interrupted deferred pass therefore had no
+    # recoverable copy at all. The snapshot is taken LAZILY, immediately before
+    # the first migration that will actually be applied, so a pass with nothing
+    # to do costs no disk and does not capture post-init state unnecessarily.
+    _snapshot_state: dict[str, object] = {"taken": dry_run}
+
+    def _ensure_snapshot() -> None:
+        if _snapshot_state["taken"]:
+            return
+        _snapshot_state["taken"] = True
+        backup_dir = _pre_migration_backup(
+            learning_db, memory_db,
+            backups_root=memory_db.parent / "pre-migration-snapshots",
+        )
+        _gc_old_backups(backup_dir)
+        details["_deferred_backup"] = str(backup_dir)
+
     blocked: set[str] = set()
     for migration in DEFERRED_MIGRATIONS:
         unmet = [d for d in migration.dependencies if d in failed or d in blocked]
@@ -529,6 +666,9 @@ def apply_deferred(
                 )
                 continue
 
+            if not dry_run and not _deferred_already_applied(conn, migration.name):
+                _ensure_snapshot()
+
             outcome, detail = _apply_single(conn, migration, dry_run=dry_run)
             details[migration.name] = detail
             if outcome == "applied":

package/src/superlocalmemory/storage/migrations/__init__.py

@@ -40,7 +40,7 @@ from . import (
 # ``superlocalmemory/storage/migrations.py`` shipped in v3.4.20 carried
 # ``CURRENT_SCHEMA_VERSION`` / ``get_schema_version`` / ``set_schema_version``
 # / ``is_v1_database`` / ``needs_migration`` / ``backup_database``. Creating
-# the ``migrations/`` package in Wave 1 shadowed that flat module, so any
+# the ``migrations/`` package (introduced in v3.4) shadowed that flat module, so any
 # caller that did ``from superlocalmemory.storage.migrations import X``
 # broke. We re-load the legacy module under a distinct name and re-export
 # the symbols here so pre-3.4.22 callers (and the pre-existing tests at

package/src/superlocalmemory/storage/migrations.py

@@ -16,6 +16,7 @@ import json
 import logging
 import shutil
 import sqlite3
+import warnings
 from datetime import UTC, datetime
 from pathlib import Path
 from typing import Any
@@ -75,7 +76,20 @@ def is_v1_database(db_path: Path) -> bool:
 
 
 def backup_database(db_path: Path) -> Path:
-    """Create timestamped backup before migration."""
+    """Create a timestamped copy of db_path alongside the source file.
+
+    .. deprecated::
+        Use the pre-migration backup in ``superlocalmemory.storage.backup``
+        instead, which produces a WAL-consistent snapshot via the SQLite
+        backup API rather than a raw filesystem copy.
+    """
+    warnings.warn(
+        "backup_database() uses shutil.copy2 which cannot produce a "
+        "consistent snapshot of a live WAL database. "
+        "Use superlocalmemory.storage.backup._pre_migration_backup() instead.",
+        DeprecationWarning,
+        stacklevel=2,
+    )
     timestamp = datetime.now(UTC).strftime("%Y%m%d_%H%M%S")
     backup_path = db_path.with_suffix(f".backup_{timestamp}.db")
     shutil.copy2(str(db_path), str(backup_path))

package/src/superlocalmemory/storage/models.py

@@ -452,3 +452,10 @@ class RecallResponse:
     # fall into (thematic context). None unless results cluster into one
     # community above threshold. Additive — backward compatible.
     community_context: dict | None = None
+    # Channels abandoned at the hang guard, so they contributed no candidates
+    # to this answer. Non-empty means the result is INCOMPLETE, not merely
+    # slow: asking the same question again on an idle machine can legitimately
+    # return something better. Empty is the normal case and the only one in
+    # which two runs of one query are expected to agree. Sorted, so the field
+    # is itself repeatable. Additive — backward compatible.
+    incomplete_channels: tuple[str, ...] = ()

package/src/superlocalmemory/storage/quantized_store.py

@@ -214,8 +214,10 @@ class QuantizedEmbeddingStore:
 
             results.append((qe.fact_id, sim))
 
-        # Sort descending by similarity
-        results.sort(key=lambda x: x[1], reverse=True)
+        # Sort by descending similarity; use fact_id as the tie-break so that
+        # two facts with identical approximate similarity always return in the
+        # same order regardless of SQLite storage layout or insertion sequence.
+        results.sort(key=lambda x: (-x[1], x[0]))
         return results[:top_k]
 
     # -- Compression helpers -----------------------------------------------

package/src/superlocalmemory/summaries/session_summary.py

@@ -8,7 +8,7 @@ CRITICAL DATA HONESTY NOTE
 ---------------------------
 On a real 3,294-fact store, only 127 facts (3.9%) carry a session_id.
 A Session Summary that presents itself as "everything you did this session"
-while silently covering 4% of the facts is the same overclaiming Wave 4
+while silently covering 4% of the facts is overclaiming — the same defect
 removed from brain/truth.py.
 
 Coverage is ALWAYS disclosed.  If the session has too few facts to summarise

package/src/superlocalmemory/ui/css/legacy-dashboard.css

@@ -202,7 +202,7 @@
             margin-bottom: 12px;
         }
 
-        /* Pane error state — WP-12 */
+        /* Pane error state */
         .pane-error {
             text-align: center;
             padding: 40px;

package/src/superlocalmemory/ui/css/neural-glass.css

@@ -1593,7 +1593,7 @@ body.ng-privacy-blur::before {
   }
 }
 
-/* WP-12 — pane-error dark-mode override */
+/* pane-error dark-mode override */
 .ng-dark .pane-error {
   color: var(--ng-status-error);
 }

package/src/superlocalmemory/ui/js/core.js

@@ -292,7 +292,7 @@ function showEmpty(containerId, icon, message) {
 }
 
 // ============================================================================
-// Pane error state — WP-12
+// Pane error state
 // showPaneError / clearPaneError / paneErrorMessage
 // ============================================================================
 

package/src/superlocalmemory/ui/js/od-brain.js

@@ -774,7 +774,7 @@
     var evb = EL('div', { className: 'card-pad' });
 
     if (registryStatus === 'error' || registryStatus === 'unknown') {
-      // Wave 4 honesty rule: failure and emptiness must not return the same value.
+      // Honesty constraint: failure and emptiness must not return the same value.
       // 'unknown' belongs HERE, not in the healthy-empty branch below. The
       // registry reader is fail-soft — a corrupt or unreadable file makes
       // _load() return {} and active_client_summary() return [], so the read