superlocalmemory 4.0.9 → 4.1.0

package/src/superlocalmemory/storage/embedding_codec.py

@@ -37,6 +37,8 @@ __all__ = [
     "EMBEDDING_BYTES",
     "encode_embedding",
     "decode_embedding",
+    "encode_float_vector",
+    "decode_float_vector",
 ]
 
 EMBEDDING_DIM: int = 768
@@ -127,3 +129,72 @@ def decode_embedding(
         f"Unexpected embedding type {type(raw).__name__!r} for fact {fact_id!r}; "
         f"expected bytes or str"
     )
+
+
+# ---------------------------------------------------------------------------
+# The same treatment for the other float vectors stored on a fact
+# ---------------------------------------------------------------------------
+#
+# A fact carries two more 768-wide vectors besides its embedding: the diagonal
+# Fisher mean and variance that the memory dynamics read. They were written as
+# JSON text, which costs about 17 KB each against 3 KB for the same numbers as
+# float32. Measured on a real 447 MB store: 116.5 MB of Fisher text describing
+# 3.6 MB of memories — thirty-two times the size of the content itself, and
+# more than a quarter of the whole file.
+#
+# The read path accepts both forms for the same reason the embedding one does:
+# a store converts when a migration reaches it, and everything has to keep
+# working in the meantime.
+
+
+def encode_float_vector(vec: list[float] | None) -> bytes | None:
+    """Serialise any float vector to a little-endian float32 BLOB."""
+    if vec is None:
+        return None
+    return np.asarray(vec, dtype=np.float32).tobytes()
+
+
+def decode_float_vector(
+    raw: bytes | str | None,
+    *,
+    field: str = "vector",
+    fact_id: str = "<unknown>",
+) -> list[float] | None:
+    """Read a float vector written as either JSON text or a float32 BLOB.
+
+    Raises rather than returning ``None`` for a malformed value: a caller
+    cannot tell a legitimately absent vector from a lost one, and these feed
+    the decay dynamics, where a silently empty vector reads as "no evidence"
+    instead of "evidence missing".
+    """
+    if raw is None or raw == "":
+        return None
+
+    if isinstance(raw, (bytes, bytearray)):
+        if len(raw) == 0 or len(raw) % 4 != 0:
+            raise ValueError(
+                f"Corrupt {field} buffer for fact {fact_id!r}: {len(raw)} bytes "
+                f"is not a multiple of 4 (float32)"
+            )
+        return np.frombuffer(raw, dtype=np.float32).tolist()
+
+    if isinstance(raw, str):
+        try:
+            value = json.loads(raw)
+        except (json.JSONDecodeError, ValueError) as exc:
+            raise ValueError(
+                f"Corrupt JSON {field} for fact {fact_id!r}: {exc}"
+            ) from exc
+        if value is None:
+            return None
+        if not isinstance(value, list):
+            raise ValueError(
+                f"{field} for fact {fact_id!r} decoded to "
+                f"{type(value).__name__}, expected a list"
+            )
+        return [float(v) for v in value]
+
+    raise ValueError(
+        f"Unexpected {field} type {type(raw).__name__!r} for fact {fact_id!r}; "
+        f"expected bytes or str"
+    )

package/src/superlocalmemory/storage/lineage_retention.py

@@ -0,0 +1,236 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under AGPL-3.0-or-later - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""Forget the provenance of things that no longer exist.
+
+`derivation_lineage` records where each derived object came from — which source
+span produced this fact, this graph edge, this scene. Every ingestion operation
+re-captures lineage for the objects present at that moment, so the table grows
+with use, and nothing has ever deleted from it.
+
+Measured on a real 447 MB store:
+
+    derivation_lineage           256,885 rows, 64 MB, plus 65 MB of indexes
+    ... describing a graph edge
+        that no longer exists    100,581 rows — 39.2% of the table
+    growth                       ~9,000 rows/day, sustained
+
+The graph is pruned. Its lineage was not, so the record of how a deleted edge
+came to exist outlives the edge forever. Those rows answer no question: the
+evidence bundle computes lineage coverage over the objects it exports, and an
+object that is gone is not exported.
+
+WHAT IS AND IS NOT DELETED
+
+Only a row whose object is provably absent — the type is one this module knows
+how to resolve, the table exists, and no row with that id is in it. An
+unrecognised object type is left alone, because "I do not know what this
+describes" is not evidence that it describes nothing.
+
+There is deliberately no age rule. Lineage is what an audit reads to answer
+"where did this come from", and a fact can be years old and still current.
+Deleting provenance for something that still exists would be destroying the
+answer while keeping the question.
+"""
+
+from __future__ import annotations
+
+import logging
+import sqlite3
+from dataclasses import dataclass, field
+
+logger = logging.getLogger(__name__)
+
+__all__ = ["OBJECT_SOURCES", "LineagePruneReport", "count_orphan_lineage",
+           "prune_orphan_lineage"]
+
+#: object_type -> (table holding it, column carrying its id).
+#: Mirrors what ``core/derivation_lineage.capture_operation_lineage`` writes.
+#: A type absent from this map is never deleted.
+OBJECT_SOURCES: dict[str, tuple[str, str]] = {
+    "fact": ("atomic_facts", "fact_id"),
+    "graph_edge": ("graph_edges", "edge_id"),
+    "memory_scene": ("memory_scenes", "scene_id"),
+    "entity_summary": ("entity_profiles", "profile_entry_id"),
+    "index_bm25": ("bm25_tokens", "fact_id"),
+    "profile": ("profiles", "profile_id"),
+}
+
+#: Rows per transaction. Big enough that commit overhead vanishes, small enough
+#: that an interrupted run has done most of its work and holds no long lock.
+_BATCH = 2_000
+
+
+class _Rows:
+    """Read and write through either a raw connection or the DatabaseManager.
+
+    The maintenance cycle holds a ``DatabaseManager``, whose lock serialises
+    every write; a migration or a test holds a plain ``sqlite3.Connection``.
+    Both are legitimate callers, and the difference is two method names.
+    """
+
+    def __init__(self, db: object) -> None:
+        self._db = db
+        self._managed = hasattr(db, "transaction") and not isinstance(
+            db, sqlite3.Connection
+        )
+
+    def query(self, sql: str, params: tuple = ()) -> list:
+        rows = self._db.execute(sql, tuple(params))
+        return list(rows) if self._managed else rows.fetchall()
+
+    def write(self, sql: str, params: tuple = ()) -> None:
+        if self._managed:
+            with self._db.transaction():
+                self._db.execute(sql, tuple(params))
+            return
+        self._db.execute("BEGIN IMMEDIATE")
+        try:
+            self._db.execute(sql, tuple(params))
+            self._db.commit()
+        except Exception:
+            self._db.rollback()
+            raise
+
+
+@dataclass(frozen=True)
+class LineagePruneReport:
+    """What was removed, by object type, and what was left alone."""
+
+    deleted: dict[str, int] = field(default_factory=dict)
+    skipped_types: tuple[str, ...] = ()
+
+    @property
+    def total(self) -> int:
+        return sum(self.deleted.values())
+
+
+def _table_exists(rows: _Rows, table: str) -> bool:
+    return bool(rows.query(
+        "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (table,)
+    ))
+
+
+def _resolvable(rows: _Rows) -> tuple[dict[str, tuple[str, str]], tuple[str, ...]]:
+    """Split the types present in the table into resolvable and not."""
+    present = {
+        str(r[0]) for r in rows.query(
+            "SELECT DISTINCT object_type FROM derivation_lineage"
+        )
+    }
+    resolvable: dict[str, tuple[str, str]] = {}
+    skipped: list[str] = []
+    for object_type in sorted(present):
+        source = OBJECT_SOURCES.get(object_type)
+        if source is None or not _table_exists(rows, source[0]):
+            skipped.append(object_type)
+            continue
+        resolvable[object_type] = source
+    return resolvable, tuple(skipped)
+
+
+def count_orphan_lineage(
+    db: object, *, profile_id: str | None = None,
+) -> LineagePruneReport:
+    """How many rows describe something absent, without deleting anything."""
+    rows = _Rows(db)
+    if not _table_exists(rows, "derivation_lineage"):
+        return LineagePruneReport()
+
+    resolvable, skipped = _resolvable(rows)
+    counts: dict[str, int] = {}
+    for object_type, (table, column) in resolvable.items():
+        sql = (
+            "SELECT COUNT(*) FROM derivation_lineage d WHERE d.object_type = ? "
+            f"AND NOT EXISTS (SELECT 1 FROM {table} t WHERE t.{column} = d.object_id)"
+        )
+        params: list[object] = [object_type]
+        if profile_id is not None:
+            sql += " AND d.profile_id = ?"
+            params.append(profile_id)
+        count = int(rows.query(sql, tuple(params))[0][0])
+        if count:
+            counts[object_type] = count
+    return LineagePruneReport(counts, skipped)
+
+
+def prune_orphan_lineage(
+    db: object,
+    *,
+    profile_id: str | None = None,
+    dry_run: bool = False,
+) -> LineagePruneReport:
+    """Delete lineage rows whose object is provably gone.
+
+    Batched and committed as it goes, so an interrupted run keeps the work it
+    already did and the next one continues. Nothing here reads a clock: what is
+    deleted depends only on what exists.
+    """
+    rows = _Rows(db)
+    if not _table_exists(rows, "derivation_lineage"):
+        return LineagePruneReport()
+
+    report = count_orphan_lineage(db, profile_id=profile_id)
+    if dry_run or not report.total:
+        return report
+
+    resolvable, skipped = _resolvable(rows)
+    deleted: dict[str, int] = {}
+
+    for object_type, (table, column) in resolvable.items():
+        if object_type not in report.deleted:
+            continue
+        removed = 0
+        while True:
+            sql = (
+                "SELECT lineage_id FROM derivation_lineage d "
+                "WHERE d.object_type = ? "
+                f"AND NOT EXISTS (SELECT 1 FROM {table} t WHERE t.{column} = d.object_id)"
+            )
+            params: list[object] = [object_type]
+            if profile_id is not None:
+                sql += " AND d.profile_id = ?"
+                params.append(profile_id)
+            sql += f" LIMIT {_BATCH}"
+
+            ids = [r[0] for r in rows.query(sql, tuple(params))]
+            if not ids:
+                break
+            placeholders = ",".join("?" * len(ids))
+            # Re-check absence in the DELETE itself. Selecting orphans and then
+            # deleting them by id is a check-then-act: another process can
+            # recreate the object between the two statements, and the row would
+            # be deleted anyway. Repeating the predicate makes the decision and
+            # the deletion one statement.
+            delete_sql = (
+                f"DELETE FROM derivation_lineage WHERE lineage_id IN ({placeholders}) "
+                f"AND NOT EXISTS (SELECT 1 FROM {table} t "
+                f"WHERE t.{column} = derivation_lineage.object_id)"
+            )
+            rows.write(delete_sql, tuple(ids))
+            # Count what survived the re-check rather than what was offered.
+            still = rows.query(
+                f"SELECT COUNT(*) FROM derivation_lineage "
+                f"WHERE lineage_id IN ({placeholders})",
+                tuple(ids),
+            )
+            remaining_here = int(still[0][0]) if still else 0
+            removed += len(ids) - remaining_here
+            if remaining_here == len(ids):
+                # Every row in this batch was rescued by a concurrent writer.
+                # Another pass would select the same rows forever.
+                break
+        if removed:
+            deleted[object_type] = removed
+            logger.info(
+                "lineage retention: removed %d row(s) describing a %s that no "
+                "longer exists", removed, object_type,
+            )
+
+    if skipped:
+        logger.info(
+            "lineage retention: left %s alone — no table is known to hold them",
+            ", ".join(skipped),
+        )
+    return LineagePruneReport(deleted, skipped)

package/src/superlocalmemory/storage/logical_edges.py

@@ -20,17 +20,58 @@ _LOGICAL_EDGE_SELECT = """
         profile_id
     FROM graph_edges
     WHERE profile_id = ?
+      AND NOT EXISTS (
+            SELECT 1 FROM atomic_facts f
+            WHERE f.fact_id IN (graph_edges.source_id, graph_edges.target_id)
+              AND NOT ({visible})
+      )
     GROUP BY profile_id, source_id, target_id, COALESCE(edge_type, 'related')
 """
 
 
+def _edge_select(conn: sqlite3.Connection) -> str:
+    """The logical-edge query, with the withheld-endpoint exclusion resolved.
+
+    WHY AN EDGE WITH A WITHHELD ENDPOINT IS NOT A LOGICAL EDGE
+    ----------------------------------------------------------
+    The retrieval channel this projection stands in for does not traverse them.
+    It loads ``graph_edges`` by scope and then prunes the result: "Edge scope
+    alone cannot authorize an endpoint. Prune both endpoints against the visible
+    fact corpus so denied facts cannot influence an allowed candidate indirectly
+    through propagation." Its entity map is filtered the same way, for a reason
+    it spells out — a withheld row carries its whole cluster's pooled entity
+    list, so it out-ranks real memories and then gets discarded at hydration,
+    spending the channel's budget on nothing.
+
+    The export predated that fix and kept the withheld endpoints. Measured on a
+    copy of the author's store: Cozo's bridge held 1,257 facts the store may not
+    return and its edges touched 805, and the graph search diverged from SQLite
+    on **every** query — three shadow checks, three mismatches. One query
+    returned 9 results against SQLite's 20, because withheld facts had taken the
+    top-k budget. The projection failed closed every time, so recall was correct
+    and the projection was dead weight.
+
+    The predicate is resolved against the passed connection because
+    ``archive_status`` and ``quarantined`` each arrive with a migration and may
+    be absent on a store the engine has not opened.
+    """
+    from superlocalmemory.storage.database import visible_fact_clause_for_connection
+
+    # The helper returns a leading-AND clause for appending; here it is needed as
+    # a standalone predicate, so strip the connective and default to "always
+    # visible" on a store that has neither column yet.
+    clause = visible_fact_clause_for_connection(conn, prefix="f").strip()
+    predicate = clause[4:].strip() if clause.upper().startswith("AND ") else clause
+    return _LOGICAL_EDGE_SELECT.format(visible=predicate or "1=1")
+
+
 def iter_logical_edges(
     conn: sqlite3.Connection, profile_id: str
 ) -> Iterator[tuple[Any, ...]]:
     """Yield normalized graph edges in deterministic fingerprint order."""
     return iter(
         conn.execute(
-            _LOGICAL_EDGE_SELECT + " ORDER BY source_id, target_id, edge_type",
+            _edge_select(conn) + " ORDER BY source_id, target_id, edge_type",
             (profile_id,),
         )
     )
@@ -39,7 +80,7 @@ def iter_logical_edges(
 def count_logical_edges(conn: sqlite3.Connection, profile_id: str) -> int:
     """Count relationships using the canonical logical identity."""
     row = conn.execute(
-        "SELECT COUNT(*) FROM (" + _LOGICAL_EDGE_SELECT + ")",
+        "SELECT COUNT(*) FROM (" + _edge_select(conn) + ")",
         (profile_id,),
     ).fetchone()
     return int(row[0] if row else 0)

package/src/superlocalmemory/storage/migration_runner.py

@@ -159,11 +159,25 @@ from superlocalmemory.storage.migrations import (
 from superlocalmemory.storage.migrations import (
     M042_correction_case_ledger as _M042,
 )
+from superlocalmemory.storage.migrations import (
+    M044_play_carries_its_own_evidence as _M044,
+)
+from superlocalmemory.storage.migrations import (
+    M045_fact_outcome_score as _M045,
+    M046_prospective_memory_has_its_own_name as _M046,
+    M047_fisher_vectors_are_stored_like_every_other_vector as _M047,
+    M048_upcoming_holds_only_what_is_upcoming as _M048,
+    M049_a_schema_version_marker_is_one_row as _M049,
+)
+from superlocalmemory.storage.migrations import (
+    M043_quarantine_display_summaries as _M043,
+)
 from superlocalmemory.storage._schema_version import (
     SUPPORTED_SCHEMA_VERSION,
     SchemaVersionError,
     check_version_or_raise as _check_version_or_raise,
     ensure_schema_version_table as _ensure_schema_version_table,
+    read_schema_version as _read_schema_version,
     write_schema_version as _write_schema_version,
 )
 from superlocalmemory.storage._migration_internals import (
@@ -249,6 +263,13 @@ MIGRATIONS: list[Migration] = [
     # contains identifiers only and does not alter temporal fact state.
     Migration(name=_M042.NAME, db_target="memory", ddl=_M042.DDL,
               dependencies=(_M032.NAME,)),
+    # M044 lets a bandit play record which memories it showed, so the reward
+    # proxy can settle it from evidence instead of always falling through to
+    # the 120-second neutral default. Additive column on M005's bandit_plays,
+    # and eager on purpose: nothing bootstraps that table at engine init, so
+    # there is no reason to defer it.
+    Migration(name=_M044.NAME, db_target="learning", ddl=_M044.DDL,
+              dependencies=(_M005.NAME,)),
     # M006 + M011 are deliberately NOT here — see DEFERRED_MIGRATIONS below.
 ]
 
@@ -312,6 +333,47 @@ DEFERRED_MIGRATIONS: list[Migration] = [
     # Main-line M034 is renumbered in V4. It must remain deferred because its
     # backfill joins engine-bootstrapped memory_scenes and atomic_facts.
     Migration(name=_M039.NAME, db_target="memory", ddl=_M039.DDL),
+    # M043 withholds model-written summaries from the retrieval corpus and
+    # un-hides the memories they displaced. Deferred because it reads and
+    # writes atomic_facts + fact_retention, both bootstrapped at engine init —
+    # the same reason M011/M013/M015/M016 are deferred. apply_deferred takes a
+    # verified snapshot before the first migration it actually applies, so the
+    # store is recoverable.
+    Migration(name=_M043.NAME, db_target="memory", ddl=_M043.DDL,
+              dependencies=(_M011.NAME,)),
+    # M045 holds the per-fact outcome score. Deferred because its backfill
+    # reads action_outcomes, which engine init bootstraps — the same reason
+    # M006 and M011 are deferred. Depends on M006 for the reward column it
+    # averages.
+    Migration(name=_M045.NAME, db_target="memory", ddl=_M045.DDL,
+              dependencies=(_M006.NAME,)),
+    # M046 renames the fact type used for planned future events, which means
+    # rebuilding atomic_facts to widen a CHECK constraint SQLite cannot alter.
+    # Deferred for the same reason as M043: atomic_facts is bootstrapped at
+    # engine init, and apply_deferred takes a verified snapshot before the first
+    # migration it applies, so a table rebuild has something to fall back to.
+    # Depends on M043 so the two never contend for the same table in one pass.
+    Migration(name=_M046.NAME, db_target="memory", ddl=_M046.DDL,
+              dependencies=(_M043.NAME,)),
+    # M047 rewrites the two Fisher vectors on each fact as float32 rather than
+    # as decimal text. Deferred because it walks every fact in atomic_facts,
+    # which engine init bootstraps. It changes no schema and both forms stay
+    # readable, so it is resumable and an interrupted store still works.
+    # Depends on M046 so a table rebuild and a full-table update never run in
+    # the same pass over the same table.
+    Migration(name=_M047.NAME, db_target="memory", ddl=_M047.DDL,
+              dependencies=(_M046.NAME,)),
+    # M048 finishes what M046 started: M046 renamed the type used for planned
+    # events without re-reading a single one of them, so the same wrongly-filed
+    # rows now carry a more confident name. Depends on M046 for the rename.
+    Migration(name=_M048.NAME, db_target="memory", ddl=_M048.DDL,
+              dependencies=(_M046.NAME,)),
+    # M049 gives schema_version the unique constraint its six writers all
+    # assumed it had. Every one uses INSERT OR IGNORE, which ignores nothing
+    # without a constraint, so each appended a duplicate per run: seven distinct
+    # versions held as 3,496 rows on one store and 234,348 on another. No
+    # dependency -- it touches a bookkeeping table no other migration reads.
+    Migration(name=_M049.NAME, db_target="memory", ddl=_M049.DDL),
 ]
 
 
@@ -579,6 +641,60 @@ def _deferred_already_applied(conn: sqlite3.Connection, name: str) -> bool:
         return False
 
 
+def _breaking_floor(learning_db: Path, memory_db: Path) -> int:
+    """Highest floor declared by a migration that is recorded complete.
+
+    A migration declares ``BREAKING_VERSION`` when a store it has touched must
+    not be opened by an older build. Only completed ones count: a migration that
+    failed has not changed anything an older build would trip over.
+    """
+    from superlocalmemory.storage._migration_internals import _MODULES
+
+    logs = {"learning": _read_log(learning_db), "memory": _read_log(memory_db)}
+    floor = 0
+    for migration in (*MIGRATIONS, *DEFERRED_MIGRATIONS):
+        module = _MODULES.get(migration.name)
+        declared = getattr(module, "BREAKING_VERSION", 0) if module else 0
+        if not declared:
+            continue
+        if logs.get(migration.db_target, {}).get(migration.name) == "complete":
+            floor = max(floor, int(declared))
+    return floor
+
+
+def _stamp_breaking_floor(
+    learning_db: Path, memory_db: Path, details: dict[str, str],
+) -> None:
+    """Raise the recorded version to the highest completed breaking floor.
+
+    Monotonic: never lowers a stored version, so it cannot undo the completion
+    certificate on an already-current store. Never fatal — a store that cannot
+    be stamped is reported, because failing the whole run here would block an
+    upgrade over a guard that only matters to older builds.
+    """
+    floor = _breaking_floor(learning_db, memory_db)
+    if floor <= 0:
+        return
+    for db_path in (learning_db, memory_db):
+        try:
+            current = _read_schema_version(db_path)
+            if current >= floor:
+                continue
+            conn = _connect(db_path)
+            try:
+                _ensure_schema_version_table(conn)
+                _write_schema_version(conn, floor)
+            finally:
+                try:
+                    conn.close()
+                except sqlite3.Error:  # pragma: no cover
+                    pass
+        except sqlite3.Error as exc:  # pragma: no cover — reported, not fatal
+            details["schema_version_floor"] = (
+                f"cannot raise the floor on {db_path}: {exc}"
+            )
+
+
 def apply_deferred(
     learning_db: Path,
     memory_db: Path,
@@ -683,6 +799,20 @@ def apply_deferred(
             except sqlite3.Error:  # pragma: no cover
                 pass
 
+    # A migration that makes the store unusable by an older build declares a
+    # floor, and that floor is written as soon as the migration is recorded
+    # complete — BEFORE and independent of the completion certificate below.
+    #
+    # The certificate is all-or-nothing across both databases by design. That is
+    # right for "is this store fully migrated" and wrong for "may an older build
+    # write to it": an unrelated failure on the other database would otherwise
+    # leave a rebuilt table guarded by the old ceiling, and the first planned
+    # event an older build stored would be rejected by the new constraint and
+    # lost. Raising the floor turns that into a refusal to start, which is what
+    # the ceiling is for.
+    if not dry_run:
+        _stamp_breaking_floor(learning_db, memory_db, details)
+
     # The version ceiling is a completion certificate, not an intent marker.
     # M039 is deferred until engine-owned tables exist, so apply_all must not
     # stamp version 39. Stamp both stores only after every eager and deferred