superlocalmemory 4.0.7 → 4.0.9

package/src/superlocalmemory/storage/embedding_codec.py

@@ -0,0 +1,129 @@
+"""Encode and decode atomic_facts.embedding values.
+
+All read and write paths for atomic_facts.embedding must go through
+``encode_embedding`` and ``decode_embedding``.  Centralising the logic here
+means a future format change requires one edit, not one per reader.
+
+Storage format
+--------------
+New rows: 768 × float32, little-endian, stored as SQLite BLOB (3,072 bytes).
+Legacy rows: JSON TEXT produced by json.dumps(list[float]).
+
+The read path accepts both formats so a partial backfill is safe by
+construction: callers see ``list[float]`` regardless of storage format.
+
+Error contract
+--------------
+A value that is neither valid JSON nor a well-formed float32 buffer raises
+``ValueError`` with the fact_id in the message.  Returning ``None`` silently
+for a malformed value is forbidden: the caller cannot distinguish a legitimate
+absent embedding from a data-loss event.
+"""
+from __future__ import annotations
+
+import json
+import logging
+from typing import TYPE_CHECKING
+
+import numpy as np
+
+logger = logging.getLogger(__name__)
+
+if TYPE_CHECKING:
+    pass
+
+__all__ = [
+    "EMBEDDING_DIM",
+    "EMBEDDING_BYTES",
+    "encode_embedding",
+    "decode_embedding",
+]
+
+EMBEDDING_DIM: int = 768
+EMBEDDING_BYTES: int = EMBEDDING_DIM * 4  # float32 = 4 bytes
+
+
+def encode_embedding(vec: list[float] | None) -> bytes | None:
+    """Serialise a float list to a binary float32 BLOB for SQLite storage.
+
+    Parameters
+    ----------
+    vec:
+        A list of floats, or ``None``.  Production embeddings are always
+        768-dimensional; the backfill script asserts the dimension before
+        calling this function.
+
+    Returns
+    -------
+    bytes | None
+        Little-endian float32 buffer, or ``None`` if *vec* is ``None``.
+    """
+    if vec is None:
+        return None
+    return np.array(vec, dtype=np.float32).tobytes()
+
+
+def decode_embedding(
+    raw: bytes | str | None,
+    *,
+    fact_id: str = "<unknown>",
+) -> list[float] | None:
+    """Deserialise an embedding from either TEXT (JSON) or BLOB (binary float32).
+
+    Parameters
+    ----------
+    raw:
+        The raw value from ``atomic_facts.embedding``:
+        ``None`` or empty string → absent embedding (returns ``None``).
+        ``bytes`` → binary float32 BLOB path.
+        ``str`` → legacy JSON TEXT path.
+    fact_id:
+        Included in any ``ValueError`` message for fast triage.
+
+    Returns
+    -------
+    list[float] | None
+        768-element list of floats, or ``None`` when the embedding is absent.
+
+    Raises
+    ------
+    ValueError
+        For a non-null value that is neither valid JSON nor a well-formed
+        float32 buffer.  Never returns ``None`` for a malformed value.
+    """
+    if raw is None or raw == "":
+        return None
+
+    if isinstance(raw, (bytes, bytearray)):
+        if len(raw) % 4 != 0 or len(raw) == 0:
+            raise ValueError(
+                f"Corrupt embedding buffer for fact {fact_id!r}: "
+                f"{len(raw)} bytes is not a multiple of 4 (float32)"
+            )
+        if len(raw) != EMBEDDING_BYTES:
+            # A torn write that happens to land on a 4-byte boundary is
+            # indistinguishable from a short vector by length alone, and it was
+            # accepted silently at debug level: 767 of 768 values still looks
+            # like a valid embedding, and every similarity computed from it is
+            # quietly wrong. Smaller vectors ARE legitimate in tests, so this is
+            # a warning rather than a refusal, but it must be visible.
+            logger.warning(
+                "embedding for fact %s is %d bytes (%d floats), not the expected "
+                "%d (%d floats) — expected only for a test vector; on a real "
+                "store this is a truncated write",
+                fact_id, len(raw), len(raw) // 4, EMBEDDING_BYTES, EMBEDDING_DIM,
+            )
+        return np.frombuffer(raw, dtype=np.float32).tolist()
+
+    if isinstance(raw, str):
+        try:
+            return json.loads(raw)
+        except (json.JSONDecodeError, ValueError) as exc:
+            raise ValueError(
+                f"Corrupt JSON embedding for fact {fact_id!r}: {exc}"
+            ) from exc
+
+    raise ValueError(
+        f"Unexpected embedding type {type(raw).__name__!r} for fact {fact_id!r}; "
+        f"expected bytes or str"
+    )

package/src/superlocalmemory/storage/embedding_migrator.py

@@ -29,6 +29,8 @@ from typing import TYPE_CHECKING, Any
 
 import numpy as np
 
+from superlocalmemory.storage.embedding_codec import encode_embedding
+
 if TYPE_CHECKING:
     from superlocalmemory.core.config import SLMConfig
 
@@ -185,7 +187,7 @@ def _activate_staged_vectors(
                 updated = conn.execute(
                     "UPDATE atomic_facts SET embedding = ? "
                     "WHERE fact_id = ? AND profile_id = ?",
-                    (embedding_json, fact_id, profile_id),
+                    (encode_embedding(vector), fact_id, profile_id),
                 )
                 if updated.rowcount != 1:
                     raise RuntimeError(
@@ -606,7 +608,7 @@ def backfill_missing_embeddings(
                 logger.warning("backfill: null vector for fact %s — skipping.", fid[:16])
                 continue
             try:
-                embedding_json = json.dumps(vec)
+                embedding_blob = encode_embedding(vec)
                 # Metadata is not an independent record: it is the pointer to
                 # a sqlite-vec row. Creating it before the vector payload leaves
                 # semantic recall permanently blind while reporting success.
@@ -638,7 +640,7 @@ def backfill_missing_embeddings(
                     # this remains the supported JSON-only fallback path.
                     db.execute(
                         "UPDATE atomic_facts SET embedding = ? WHERE fact_id = ?",
-                        (embedding_json, fid),
+                        (embedding_blob, fid),
                     )
                 except Exception:
                     if projection_written and vector_store is not None:

package/src/superlocalmemory/storage/migration_runner.py

@@ -4,8 +4,6 @@
 
 """Forward-only additive migrations for SLM v3.4.22.
 
-LLD reference: ``.backup/active-brain/lld/LLD-07-schema-migrations-and-security-primitives.md``
-Section 4 (Migration Runner).
 
 Contract:
   - ``apply_all(learning_db, memory_db, *, dry_run=False) -> dict`` —
@@ -34,6 +32,7 @@ catalogue and the public orchestration functions.
 from __future__ import annotations
 
 import logging
+import os
 import sqlite3
 from pathlib import Path
 
@@ -177,6 +176,10 @@ from superlocalmemory.storage._migration_internals import (
     _migration_log_exists,
     _read_log,
 )
+from superlocalmemory.storage.backup import (
+    _gc_old_backups,
+    _pre_migration_backup,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -375,6 +378,61 @@ def _bootstrap_learning_schema(learning_db: Path, *, dry_run: bool) -> str | Non
     return None
 
 
+def _foreign_live_daemon(memory_db: Path) -> "int | None":
+    """Return the pid of another live daemon holding this data dir, or None.
+
+    Migrations are not fenced against concurrent writers. The realistic hazard
+    is an OLD daemon still running after an upgrade while a NEW one starts: its
+    WAL appends continue while DDL is applied, which can make a migration fail
+    non-deterministically. The snapshot itself stays consistent — the SQLite
+    backup API copies committed pages only — and a racing migration is recorded
+    as ``failed`` and is non-fatal, so this does not corrupt data.
+
+    This detects the condition and reports it. It deliberately does NOT refuse:
+    ``apply_all`` runs inside the daemon's own startup, so refusing whenever "a
+    daemon is running" would refuse on itself, and blocking on a lock here would
+    risk wedging startup — a worse outcome than a retryable failed step.
+    """
+    try:
+        pid_file = memory_db.parent / "daemon.pid"
+        if not pid_file.is_file():
+            return None
+        pid = int(pid_file.read_text().strip() or 0)
+        if pid <= 0 or pid == os.getpid():
+            return None
+        os.kill(pid, 0)          # signal 0 tests liveness without touching it
+        return pid
+    except (OSError, ValueError):
+        return None
+
+
+def _nothing_left_to_apply(learning_db: Path, memory_db: Path) -> bool:
+    """True when every migration is already recorded in its target database.
+
+    Used to decide whether a snapshot is worth taking. A snapshot is only
+    valuable when something is about to change; taking one on a start where
+    nothing changes copies the ALREADY-MIGRATED store and then prunes a
+    generation — so after two such starts the last copy of the original is gone,
+    and the safety net has quietly deleted the thing it exists to protect.
+
+    Errs toward False, which means "take the snapshot" — the safe direction.
+    """
+    try:
+        for migration in MIGRATIONS:
+            db_path = _db_for(migration.db_target, learning_db, memory_db)
+            if not db_path.exists():
+                return False
+            conn = _connect(db_path)
+            try:
+                if not _deferred_already_applied(conn, migration.name):
+                    return False
+            finally:
+                conn.close()
+    except Exception:  # noqa: BLE001 — any doubt means take the snapshot
+        return False
+    return True
+
+
 def apply_all(
     learning_db: Path,
     memory_db: Path,
@@ -401,6 +459,44 @@ def apply_all(
     failed: list[str] = []
     details: dict[str, str] = {}
 
+    # Take a consistent snapshot of both databases before any migration runs.
+    # The backup uses the SQLite backup API so in-flight WAL writers are
+    # never captured mid-transaction. InsufficientDiskSpaceError propagates
+    # to the caller — migration is intentionally aborted when disk is too
+    # tight to keep a recoverable copy.
+    # A snapshot is only worth taking when something is about to change. This
+    # runs on every engine construction, not just upgrades, so snapshotting
+    # unconditionally meant an ordinary start copied the already-migrated store
+    # and pruned a generation — two extra starts and the original was gone.
+    _pending = not _nothing_left_to_apply(learning_db, memory_db)
+    if not dry_run and not _pending:
+        details["_backup"] = "skipped: every migration already applied"
+
+    if not dry_run and _pending:
+        _other = _foreign_live_daemon(memory_db)
+        if _other is not None:
+            logger.warning(
+                "Another SuperLocalMemory daemon (pid %s) is still running and "
+                "writing to this data directory. Migrations are not fenced "
+                "against concurrent writers, so a step may fail and need a "
+                "retry. Your data is not at risk: the snapshot copies committed "
+                "pages only, and a failed step is recorded, never forced. Stop "
+                "the other daemon and restart if a step fails.",
+                _other,
+            )
+            details["_concurrent_daemon_pid"] = str(_other)
+
+        backup_dir = _pre_migration_backup(
+            learning_db, memory_db,
+            backups_root=memory_db.parent / "pre-migration-snapshots",
+        )
+        # _pre_migration_backup returns the snapshots root itself, so this is
+        # the directory to prune. Passing .parent pointed the collector at the
+        # data directory, where it matched nothing and pruned nothing — leaving
+        # every snapshot on disk for ever.
+        _gc_old_backups(backup_dir)
+        details["_backup"] = str(backup_dir)
+
     schema_error = _bootstrap_learning_schema(learning_db, dry_run=dry_run)
     if schema_error is not None:
         failed.append("learning_schema_bootstrap")
@@ -461,6 +557,28 @@ def apply_all(
     }
 
 
+def _deferred_already_applied(conn: sqlite3.Connection, name: str) -> bool:
+    """True when ``name`` is recorded as ``complete`` in this database's migration_log.
+
+    Used only to decide whether a snapshot is needed. On any error it returns
+    False, which errs toward taking a snapshot — the safe direction.
+
+    A row whose status is ``failed`` or ``in_progress`` is NOT considered applied:
+    the runner will retry those entries, and the store deserves a fresh snapshot
+    before any retry runs DDL against it.  Counting any row (regardless of status)
+    caused ``_nothing_left_to_apply`` to return True after a failed migration,
+    so the retry ran against the already-partial store with no new safety copy.
+    """
+    try:
+        row = conn.execute(
+            "SELECT 1 FROM migration_log WHERE name = ? AND status = 'complete' LIMIT 1",
+            (name,),
+        ).fetchone()
+        return row is not None
+    except sqlite3.Error:
+        return False
+
+
 def apply_deferred(
     learning_db: Path,
     memory_db: Path,
@@ -495,6 +613,25 @@ def apply_deferred(
     failed: list[str] = []
     details: dict[str, str] = {}
 
+    # apply_all snapshots before it touches anything; this pass did not, yet it
+    # applies real DDL to both managed databases — including the column the
+    # daemon needs to start. An interrupted deferred pass therefore had no
+    # recoverable copy at all. The snapshot is taken LAZILY, immediately before
+    # the first migration that will actually be applied, so a pass with nothing
+    # to do costs no disk and does not capture post-init state unnecessarily.
+    _snapshot_state: dict[str, object] = {"taken": dry_run}
+
+    def _ensure_snapshot() -> None:
+        if _snapshot_state["taken"]:
+            return
+        _snapshot_state["taken"] = True
+        backup_dir = _pre_migration_backup(
+            learning_db, memory_db,
+            backups_root=memory_db.parent / "pre-migration-snapshots",
+        )
+        _gc_old_backups(backup_dir)
+        details["_deferred_backup"] = str(backup_dir)
+
     blocked: set[str] = set()
     for migration in DEFERRED_MIGRATIONS:
         unmet = [d for d in migration.dependencies if d in failed or d in blocked]
@@ -529,6 +666,9 @@ def apply_deferred(
                 )
                 continue
 
+            if not dry_run and not _deferred_already_applied(conn, migration.name):
+                _ensure_snapshot()
+
             outcome, detail = _apply_single(conn, migration, dry_run=dry_run)
             details[migration.name] = detail
             if outcome == "applied":

package/src/superlocalmemory/storage/migrations/__init__.py

@@ -40,7 +40,7 @@ from . import (
 # ``superlocalmemory/storage/migrations.py`` shipped in v3.4.20 carried
 # ``CURRENT_SCHEMA_VERSION`` / ``get_schema_version`` / ``set_schema_version``
 # / ``is_v1_database`` / ``needs_migration`` / ``backup_database``. Creating
-# the ``migrations/`` package in Wave 1 shadowed that flat module, so any
+# the ``migrations/`` package (introduced in v3.4) shadowed that flat module, so any
 # caller that did ``from superlocalmemory.storage.migrations import X``
 # broke. We re-load the legacy module under a distinct name and re-export
 # the symbols here so pre-3.4.22 callers (and the pre-existing tests at

package/src/superlocalmemory/storage/migrations.py

@@ -16,6 +16,7 @@ import json
 import logging
 import shutil
 import sqlite3
+import warnings
 from datetime import UTC, datetime
 from pathlib import Path
 from typing import Any
@@ -75,7 +76,20 @@ def is_v1_database(db_path: Path) -> bool:
 
 
 def backup_database(db_path: Path) -> Path:
-    """Create timestamped backup before migration."""
+    """Create a timestamped copy of db_path alongside the source file.
+
+    .. deprecated::
+        Use the pre-migration backup in ``superlocalmemory.storage.backup``
+        instead, which produces a WAL-consistent snapshot via the SQLite
+        backup API rather than a raw filesystem copy.
+    """
+    warnings.warn(
+        "backup_database() uses shutil.copy2 which cannot produce a "
+        "consistent snapshot of a live WAL database. "
+        "Use superlocalmemory.storage.backup._pre_migration_backup() instead.",
+        DeprecationWarning,
+        stacklevel=2,
+    )
     timestamp = datetime.now(UTC).strftime("%Y%m%d_%H%M%S")
     backup_path = db_path.with_suffix(f".backup_{timestamp}.db")
     shutil.copy2(str(db_path), str(backup_path))

package/src/superlocalmemory/storage/models.py

@@ -452,3 +452,10 @@ class RecallResponse:
     # fall into (thematic context). None unless results cluster into one
     # community above threshold. Additive — backward compatible.
     community_context: dict | None = None
+    # Channels abandoned at the hang guard, so they contributed no candidates
+    # to this answer. Non-empty means the result is INCOMPLETE, not merely
+    # slow: asking the same question again on an idle machine can legitimately
+    # return something better. Empty is the normal case and the only one in
+    # which two runs of one query are expected to agree. Sorted, so the field
+    # is itself repeatable. Additive — backward compatible.
+    incomplete_channels: tuple[str, ...] = ()

package/src/superlocalmemory/storage/quantized_store.py

@@ -214,8 +214,10 @@ class QuantizedEmbeddingStore:
 
             results.append((qe.fact_id, sim))
 
-        # Sort descending by similarity
-        results.sort(key=lambda x: x[1], reverse=True)
+        # Sort by descending similarity; use fact_id as the tie-break so that
+        # two facts with identical approximate similarity always return in the
+        # same order regardless of SQLite storage layout or insertion sequence.
+        results.sort(key=lambda x: (-x[1], x[0]))
         return results[:top_k]
 
     # -- Compression helpers -----------------------------------------------

package/src/superlocalmemory/summaries/base.py

@@ -98,6 +98,165 @@ GENERATED_BY_LLM_C = "llm_c"
 """Cloud LLM (Mode C).  Falls back via llm_b to extractive."""
 
 
+# ── highlight formatting ────────────────────────────────────────────────────
+
+#: Display width for one bullet in a summary body.
+#:
+#: Chosen for a bullet, not for a paragraph. The generators originally truncated
+#: at 300 characters and nothing else, which looks fine on a synthetic corpus of
+#: one-line facts and falls apart on a real store: agent-written facts routinely
+#: contain blank lines and markdown headings, so a 300-character slice rendered
+#: as six or more display lines and the bullet list stopped being a list.
+HIGHLIGHT_CHARS = 180
+
+
+def format_highlight(content: str, limit: int = HIGHLIGHT_CHARS) -> str:
+    """Collapse *content* to a single readable line for a summary bullet.
+
+    Three things, in order:
+
+    1. **Flatten whitespace.** Newlines, blank lines and runs of spaces all
+       become one space. This is the fix that matters: character truncation
+       alone cannot keep a multi-paragraph fact on one line, and every
+       generator here writes into a bullet list.
+    2. **Prefer a whole first sentence** when there is one and it fits. A
+       complete sentence reads better than a slice of one, and the first
+       sentence of a report is usually its summary.
+    3. **Otherwise cut at a word boundary** and mark the cut with an ellipsis,
+       so it is visible that text was dropped rather than that a fact ended
+       mid-word.
+
+    Markdown heading markers are stripped because a flattened ``**Summary**``
+    mid-sentence reads as noise.
+    """
+    import re
+
+    text = re.sub(r"\s+", " ", (content or "")).strip()
+    # Leading/inline markdown emphasis and heading marks, once flattened, add
+    # nothing but clutter to a one-line bullet.
+    text = re.sub(r"(?:^|\s)#{1,6}\s+", " ", text)
+    text = re.sub(r"\*\*(.+?)\*\*", r"\1", text)
+    text = re.sub(r"\s+", " ", text).strip()
+
+    if not text:
+        return ""
+    if len(text) <= limit:
+        return text
+
+    # A complete first sentence, if it fits comfortably.
+    match = re.match(r"(.+?[.!?])(?:\s|$)", text)
+    if match:
+        sentence = match.group(1).strip()
+        if len(sentence) <= limit:
+            return sentence
+
+    cut = text[:limit]
+    space = cut.rfind(" ")
+    if space > limit * 0.6:          # don't cut a long unbroken token to a stub
+        cut = cut[:space]
+    return cut.rstrip(" ,;:—-") + "…"
+
+
+# ── LLM output cleanup ──────────────────────────────────────────────────────
+
+#: Sentences a chat-tuned model emits *around* the answer rather than as part of
+#: it. Anchored to the start of a paragraph so they cannot match mid-content.
+#:
+#: Measured, not guessed: a Mode B summary on the author's own store opened with
+#: "I apologize for the previous confusion. It seems that I misunderstood the
+#: context of the texts provided.\n\nTo provide a concise summary paragraph, here
+#: is a merge of all the key information:" — 180 characters of the model talking
+#: to itself, shown to the user as their daily reflection.
+_LLM_PREAMBLE = (
+    r"^(?:"
+    r"i\s+apologi[sz]e\b.*"
+    r"|i'?m\s+sorry\b.*"
+    r"|it\s+seems\s+that\s+i\b.*"
+    r"|sure[,!.]?\s*(?:thing)?\b.*"
+    r"|certainly[,!.]?\b.*"
+    r"|of\s+course[,!.]?\b.*"
+    r"|here(?:'s|\s+is|\s+are)\b[^.!?]*:"
+    r"|to\s+(?:provide|summari[sz]e|answer)\b[^.!?]*:"
+    r"|based\s+on\s+the\s+(?:facts|texts|information|data)\s+provided[,:]?"
+    r"|as\s+(?:an|a)\s+(?:ai|language\s+model)\b.*"
+    r")\s*$"
+)
+
+#: Closing pleasantries. Same anchoring rule.
+_LLM_POSTAMBLE = (
+    r"^(?:"
+    r"(?:i\s+hope|hope)\s+(?:this|that)\s+helps\b.*"
+    r"|let\s+me\s+know\b.*"
+    r"|feel\s+free\s+to\b.*"
+    r"|would\s+you\s+like\s+me\s+to\b.*"
+    r")$"
+)
+
+
+def clean_llm_summary(text: str) -> str:
+    """Strip chat-assistant scaffolding from a model-written summary.
+
+    WHY THIS EXISTS
+    ---------------
+    Mode B/C summaries are shown to the user as *their* memory, with no chat
+    framing around them. A chat-tuned model does not know that: it opens with an
+    apology or "Here is a concise summary:" and closes with "Let me know if you
+    want more detail". Both are addressed to a conversation that the reader
+    cannot see, and both make the product look broken.
+
+    The system prompt now asks for bare prose, which handles most of it. This is
+    the second line of defence, because instruction-following on a 3B local
+    model is not something to bet the displayed output on.
+
+    Conservative by construction: patterns are anchored to whole paragraphs or
+    whole leading sentences, so a summary that legitimately contains the word
+    "sure" mid-paragraph is untouched. If stripping would empty the text, the
+    original is returned — a scaffolded summary beats a blank one.
+    """
+    import re
+
+    original = (text or "").strip()
+    if not original:
+        return ""
+
+    # Fenced code blocks wrapping the whole answer: keep the contents.
+    fenced = re.match(r"^```[a-zA-Z]*\n(.*?)\n?```$", original, re.DOTALL)
+    if fenced:
+        original = fenced.group(1).strip()
+
+    paras = [p.strip() for p in re.split(r"\n\s*\n", original) if p.strip()]
+
+    while paras and re.match(_LLM_PREAMBLE, paras[0], re.IGNORECASE | re.DOTALL):
+        paras.pop(0)
+    while paras and re.match(_LLM_POSTAMBLE, paras[-1], re.IGNORECASE | re.DOTALL):
+        paras.pop()
+
+    if not paras:
+        return original
+
+    # A preamble that shares a paragraph with real content: drop just the leading
+    # sentence, and only when what follows is substantial enough to stand alone.
+    lead = re.match(r"(.+?[.:!?])\s+(\S.*)$", paras[0], re.DOTALL)
+    if lead and re.match(_LLM_PREAMBLE, lead.group(1), re.IGNORECASE | re.DOTALL):
+        if len(lead.group(2).strip()) > 40:
+            paras[0] = lead.group(2).strip()
+
+    cleaned = "\n\n".join(paras).strip()
+    return cleaned or original
+
+
+#: System prompt for every summary generator, Mode B and Mode C alike.
+#:
+#: Mode C always sent one; Mode B sent none at all, which is why local-model
+#: output arrived wrapped in chat scaffolding while cloud output did not.
+SUMMARY_SYSTEM_PROMPT = (
+    "You summarise a person's own saved notes for them. "
+    "Reply with the summary text only — no preamble, no apologies, no sign-off, "
+    "no markdown headings, and never refer to yourself or to these instructions. "
+    "Write plain prose in the third person about the work described."
+)
+
+
 def get_mode_str(config: object | None) -> str:
     """Extract the operating mode string ('a', 'b', or 'c') from a config."""
     if config is None: