superlocalmemory 4.1.13 → 4.1.14

package/src/superlocalmemory/storage/migrations/M047_fisher_vectors_are_stored_like_every_other_vector.py

@@ -1,306 +0,0 @@
-# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
-# Licensed under AGPL-3.0-or-later - see LICENSE file
-# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
-
-"""Rewrite the two Fisher vectors on each fact as float32, like the embedding.
-
-WHAT THIS IS ABOUT
-
-A fact carries three vectors of the same width: its embedding, and the diagonal
-Fisher mean and variance the memory dynamics read to decide how fast it decays.
-4.0.9 converted the embedding from JSON text to float32 and got 5.5x. The other
-two were left in text.
-
-Measured on a real 447 MB store:
-
-    fisher_mean + fisher_variance    116.5 MB
-    embedding (already binary)        15.5 MB
-    content — the memories themselves  3.6 MB
-
-The Fisher matrices were thirty-two times the size of the memories they
-describe, and more than a quarter of the entire file, because a float printed
-as decimal text costs about 22 bytes and the same float costs 4.
-
-WHY AN UPDATE AND NOT A REBUILD
-
-``atomic_facts`` has a TEXT primary key, so it also has an implicit rowid, and
-its full-text index is external-content keyed on that rowid. Rebuilding the
-table reassigns rowids and every index entry then points at a different fact —
-searches keep working and return the wrong rows. An UPDATE touches no rowid, so
-the search index is untouched by construction. Nothing here alters the schema:
-the columns are declared TEXT and SQLite stores a BLOB in them regardless.
-
-WHY IT CAN BE INTERRUPTED
-
-Every row is converted independently and the read path accepts both forms, so a
-store stopped halfway is a working store. Re-running finishes it. That is why
-the work is committed in batches rather than held in one transaction across
-three thousand facts.
-
-WHAT VERIFY CAN AND CANNOT CHECK
-
-float32 has about seven significant digits, and these numbers were written from
-float64, so the conversion loses precision on purpose. That trade is checked in
-the tests, which hold the original values and compare against them at the
-tolerance float32 provides.
-
-``verify`` runs after the fact, when the text it replaced no longer exists, so
-it cannot make that comparison. What it can do is refuse the shapes a broken
-conversion would leave: a convertible vector still in the old form, meaning the
-run stopped early; a buffer that is not a whole number of floats; a non-finite
-value; and a buffer that is entirely zero — which passes every structural check
-and is a deletion wearing the right shape, because the dynamics read an all-zero
-vector as "this memory carries no evidence".
-"""
-
-from __future__ import annotations
-
-import json
-import logging
-import sqlite3
-
-import numpy as np
-
-logger = logging.getLogger(__name__)
-
-NAME = "M047_fisher_vectors_are_stored_like_every_other_vector"
-DB_TARGET = "memory"
-
-#: No schema change and both forms are readable, so an older build opening a
-#: converted store is safe. The floor does not move.
-BREAKING_VERSION = 0
-
-_TABLE = "atomic_facts"
-_COLUMNS = ("fisher_mean", "fisher_variance")
-
-#: Rows per transaction. Large enough that the commit overhead is irrelevant,
-#: small enough that an interrupted run loses a fraction of a second of work.
-_BATCH = 500
-
-#: float32 keeps about seven significant digits. Anything inside this is the
-#: cost that was chosen; anything outside it is a bug.
-_TOLERANCE = 1e-6
-
-DDL = """
--- No schema change. apply() rewrites the two Fisher columns of each fact from
--- JSON text to a little-endian float32 buffer, in place, in batches.
-"""
-
-
-def _is_text_vector(value: object) -> bool:
-    """A value still in the old form: a string that opens like a JSON list."""
-    return isinstance(value, str) and value.strip().startswith("[")
-
-
-def _encode(text: str) -> bytes | None:
-    """Text form to float32 buffer, or None when there is nothing to store."""
-    parsed = json.loads(text)
-    if parsed is None:
-        return None
-    if not isinstance(parsed, list):
-        raise ValueError(f"expected a list, got {type(parsed).__name__}")
-    return np.asarray(parsed, dtype=np.float32).tobytes()
-
-
-def _pending(conn: sqlite3.Connection) -> int:
-    """How many rows still hold a Fisher vector in the old form."""
-    clauses = " OR ".join(
-        f"(typeof({col}) = 'text' AND {col} LIKE '[%')" for col in _COLUMNS
-    )
-    row = conn.execute(f"SELECT COUNT(*) FROM {_TABLE} WHERE {clauses}").fetchone()
-    return int(row[0]) if row else 0
-
-
-def apply(conn: sqlite3.Connection) -> None:
-    """Convert every remaining text Fisher vector, in batches, resumably."""
-    existing = {r[1] for r in conn.execute(f"PRAGMA table_info({_TABLE})")}
-    missing = [c for c in _COLUMNS if c not in existing]
-    if missing:
-        logger.info("M047: %s has no %s; nothing to convert", _TABLE, missing)
-        return
-
-    remaining = _pending(conn)
-    if remaining == 0:
-        logger.info("M047: no Fisher vector is in the old form")
-        return
-    logger.info("M047: converting Fisher vectors on %d fact(s)", remaining)
-
-    converted = 0
-    skipped = 0
-    # Walk forward by rowid rather than re-querying "what is left". A row can
-    # legitimately still match the WHERE clause after being visited — one of its
-    # two vectors may be unreadable while the other converts — and re-selecting
-    # on the predicate alone would then hand back the same row forever.
-    cursor = 0
-    clauses = " OR ".join(
-        f"(typeof({col}) = 'text' AND {col} LIKE '[%')" for col in _COLUMNS
-    )
-
-    while True:
-        batch = conn.execute(
-            f"SELECT rowid, fact_id, {', '.join(_COLUMNS)} FROM {_TABLE} "
-            f"WHERE rowid > ? AND ({clauses}) ORDER BY rowid LIMIT {_BATCH}",
-            (cursor,),
-        ).fetchall()
-        if not batch:
-            break
-        cursor = batch[-1][0]
-
-        updates: list[tuple[object, ...]] = []
-        for row in batch:
-            rowid, fact_id = row[0], row[1]
-            values: list[object] = []
-            changed = False
-            unreadable = False
-            # The two vectors are independent. One being unreadable is no reason
-            # to leave the other in the form that costs five times as much.
-            for offset, column in enumerate(_COLUMNS, start=2):
-                raw = row[offset]
-                if not _is_text_vector(raw):
-                    values.append(raw)
-                    continue
-                try:
-                    values.append(_encode(raw))
-                    changed = True
-                except (ValueError, TypeError, json.JSONDecodeError) as exc:
-                    # A vector that cannot be parsed is left exactly as it is.
-                    # Replacing it with NULL would turn "unreadable" into
-                    # "absent", and absent reads as "no evidence" in the decay
-                    # dynamics rather than as something to look at.
-                    logger.warning(
-                        "M047: leaving %s on fact %s as text — %s",
-                        column, fact_id, exc,
-                    )
-                    values.append(raw)
-                    unreadable = True
-            if unreadable:
-                skipped += 1
-            if changed:
-                updates.append((*values, rowid))
-
-        if updates:
-            conn.execute("BEGIN IMMEDIATE")
-            try:
-                # Only write over a value that is STILL in the old form. Reading
-                # a row and writing it back later is a lost update: a live
-                # writer that recomputes this fact's vectors between the two
-                # would have its new value replaced by the old one re-encoded,
-                # silently, with no error and no retry. The typeof() guard makes
-                # the update conditional on nothing having changed underneath.
-                for column in _COLUMNS:
-                    per_column = [
-                        (values[_COLUMNS.index(column)], rowid)
-                        for *values, rowid in updates
-                    ]
-                    conn.executemany(
-                        f"UPDATE {_TABLE} SET {column} = ? "
-                        f"WHERE rowid = ? AND typeof({column}) = 'text'",
-                        per_column,
-                    )
-                conn.commit()
-            except Exception:
-                conn.rollback()
-                raise
-            converted += len(updates)
-
-    logger.info(
-        "M047: converted %d fact(s); left %d vector(s) as text for inspection",
-        converted, skipped,
-    )
-
-
-def repair(conn: sqlite3.Connection) -> None:
-    """Finish the conversion on a store that has drifted back to the old form.
-
-    WHY A COMPLETED MIGRATION NEEDS THIS
-    ------------------------------------
-    A long-running daemon holds its writers in memory. This migration and the
-    write-path change that stores the new form shipped in the same commit, so a
-    daemon started before that commit converts nothing and keeps writing text --
-    and the migration, run by some later short-lived process, marks itself
-    complete against a store that is still gaining old-form rows behind it.
-
-    That happened, and the numbers say so exactly: on the author's store the
-    migration completed at 08:37, the newest converted row was written at 08:08,
-    and sixteen facts written between 10:50 and 17:34 were text. The daemon had
-    been up for twenty-five hours, since before the commit existed.
-
-    Without a repair hook the framework refuses to touch a completed migration
-    and logs "schema incomplete for completed migration; automatic replay is
-    disabled" on every open. Both halves of that are wrong here -- the
-    conversion did not stop early, and disabling replay for a false alarm means
-    a migration that IS incomplete later goes unrepaired. The mechanism was
-    already there; this migration simply never supplied the hook.
-
-    ``apply`` is the whole repair: every row converts independently, the read
-    path accepts both forms, and re-running finishes what is left. So this is
-    not a second implementation to keep in step -- it is the same one.
-    """
-    apply(conn)
-
-
-def verify(conn: sqlite3.Connection) -> bool:
-    """Every convertible vector is converted, and the numbers still agree.
-
-    Reading a sample back and comparing it to what it replaced is the only
-    check that distinguishes a conversion from a deletion. A count of BLOBs
-    would pass just as happily on zeroed buffers.
-    """
-    existing = {r[1] for r in conn.execute(f"PRAGMA table_info({_TABLE})")}
-    if any(c not in existing for c in _COLUMNS):
-        return True
-
-    # A row whose vector could not be parsed is left as text on purpose, so
-    # "some text remains" is not by itself a failure. What would be a failure is
-    # text that WOULD have converted — that means apply() stopped early.
-    for column in _COLUMNS:
-        leftover = conn.execute(
-            f"SELECT fact_id, {column} FROM {_TABLE} "
-            f"WHERE typeof({column}) = 'text' AND {column} LIKE '[%'"
-        ).fetchall()
-        for fact_id, raw in leftover:
-            try:
-                _encode(raw)
-            except (ValueError, TypeError, json.JSONDecodeError):
-                continue  # genuinely unconvertible; apply() reported it
-            logger.error(
-                "M047 verify: %s on fact %s is still text and would have "
-                "converted, so the conversion stopped early",
-                column, fact_id,
-            )
-            return False
-
-    for column in _COLUMNS:
-        sample = conn.execute(
-            f"SELECT {column} FROM {_TABLE} "
-            f"WHERE typeof({column}) = 'blob' LIMIT 50"
-        ).fetchall()
-        nonzero = 0
-        for (raw,) in sample:
-            if raw is None:
-                continue
-            if len(raw) == 0 or len(raw) % 4 != 0:
-                logger.error(
-                    "M047 verify: a %s buffer is %d bytes, not a float32 vector",
-                    column, len(raw),
-                )
-                return False
-            values = np.frombuffer(raw, dtype=np.float32)
-            if not np.all(np.isfinite(values)):
-                logger.error("M047 verify: a %s buffer holds a non-finite value", column)
-                return False
-            if np.any(values):
-                nonzero += 1
-        if sample and nonzero == 0:
-            # EVERY sampled vector being zero is a blanket wipe wearing the
-            # right shape: the buffers are the correct length and finite, and
-            # the dynamics would read all of them as "this memory carries no
-            # evidence". One honestly-zero vector is not that, and refusing on
-            # a single one would mark the migration incomplete forever on a
-            # store that legitimately holds one.
-            logger.error(
-                "M047 verify: every sampled %s buffer is entirely zero, which "
-                "is a wipe rather than a conversion", column,
-            )
-            return False
-    return True

package/src/superlocalmemory/storage/migrations/M048_upcoming_holds_only_what_is_upcoming.py

@@ -1,229 +0,0 @@
-# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
-# Licensed under AGPL-3.0-or-later - see LICENSE file
-# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
-
-"""Re-read what is filed as a plan, and file the rest correctly.
-
-M046 renamed the type used for planned events. Renaming is all it did: every
-row that said ``temporal`` now says ``prospective``, and the contents were never
-re-read. On a real store 869 rows carried that type and seven of them contained
-any planning language — session summaries, records of finished work, lists of
-commits. After the rename that same set carries a more confident name, and the
-question a user actually asks — "what is coming up" — reads exactly that set.
-
-So the rename needs a second half. This one re-reads each of those memories
-under the rule that decides the question today and moves the ones that are not
-plans to the type their wording supports.
-
-WHY THIS ONLY DEMOTES
-
-It never promotes. A memory filed as ordinary that turns out to read like a plan
-is left alone, because the cost is asymmetric in the same way the classifier's
-own tiers are: a plan filed as an ordinary memory is still found by every
-retrieval channel, and an ordinary memory filed as a plan is the pollution this
-exists to remove. Walking every fact in the store to look for promotions would
-also cost a full table scan for the smaller half of the benefit.
-
-WHY IT IS SAFE TO RE-RUN
-
-The rule is a pure function of the text. Running it twice gives the same answer,
-so a second pass moves nothing.
-"""
-
-from __future__ import annotations
-
-import logging
-import contextlib
-import sqlite3
-from typing import Any
-
-logger = logging.getLogger(__name__)
-
-NAME = "M048_upcoming_holds_only_what_is_upcoming"
-DB_TARGET = "memory"
-
-#: No schema change and no new value — every type written here already existed.
-BREAKING_VERSION = 0
-
-_TABLE = "atomic_facts"
-_PROSPECTIVE = "prospective"
-
-#: Rows per transaction.
-_BATCH = 500
-
-DDL = """
--- No schema change. apply() re-reads the content of every fact typed
--- 'prospective' and demotes the ones whose wording does not describe something
--- still ahead.
-"""
-
-
-def _resolve(content: str) -> str:
-    """The type this text supports — and the question asked in the right order.
-
-    The full classifier answers "which of the four is this", and it asks about
-    opinion before it asks about plans. That is right for a new memory: "I think
-    we should ship next week" is an opinion. It is WRONG here, because this
-    migration is only deciding whether a row belongs under the plan type, and
-    "we should deploy next Tuesday" IS something coming up. Asking the full
-    classifier demoted real deadlines into opinions and they vanished from the
-    list of what is upcoming — which is the exact harm this exists to repair.
-
-    So: if it reads as a plan, it stays a plan. Only when it does not is the
-    full classifier asked where it should go instead, and its answer is taken
-    only if it is not "prospective".
-    """
-    from superlocalmemory.encoding.fact_extractor import _classify_sentence
-    from superlocalmemory.encoding.prospective_markers import looks_prospective
-
-    text = content or ""
-    if looks_prospective(text):
-        return _PROSPECTIVE
-    resolved = _classify_sentence(text).value
-    return "semantic" if resolved == _PROSPECTIVE else resolved
-
-
-@contextlib.contextmanager
-def _held(conn: sqlite3.Connection):
-    """Yield the connection the caller already owns, and leave it open."""
-    yield conn
-
-
-def apply(
-    conn: sqlite3.Connection | None = None,
-    *,
-    open_connection: Any = None,
-) -> None:
-    """Demote every wrongly-filed plan, in batches, resumably.
-
-    Pass ``conn`` when the caller owns the connection for the whole pass -- the
-    migration runner at startup, where nothing else is writing. Pass
-    ``open_connection`` (a context manager factory, typically the database
-    manager's ``raw_connection``) on a running store: each batch then takes and
-    releases the process write lock, so a memory being saved waits one batch
-    rather than the whole pass. The batching below cannot do that on its own,
-    because holding the connection is what holds the lock.
-    """
-    if (conn is None) == (open_connection is None):
-        raise ValueError("pass exactly one of conn or open_connection")
-    acquire = (lambda: _held(conn)) if conn is not None else open_connection
-
-    with acquire() as probe:
-        existing = {r[1] for r in probe.execute(f"PRAGMA table_info({_TABLE})")}
-    if "fact_type" not in existing or "content" not in existing:
-        logger.info("M048: %s has no fact_type/content; nothing to re-read", _TABLE)
-        return
-
-    cursor = 0
-    demoted = 0
-    kept = 0
-    while True:
-        with acquire() as active:
-            batch = active.execute(
-                f"SELECT rowid, fact_id, content FROM {_TABLE} "
-                f"WHERE rowid > ? AND fact_type = ? ORDER BY rowid LIMIT {_BATCH}",
-                (cursor, _PROSPECTIVE),
-            ).fetchall()
-            if not batch:
-                break
-            cursor = batch[-1][0]
-
-            moves: list[tuple[str, int]] = []
-            for rowid, _fact_id, content in batch:
-                resolved = _resolve(content)
-                if resolved == _PROSPECTIVE:
-                    kept += 1
-                    continue
-                moves.append((resolved, rowid))
-
-            if moves:
-                active.execute("BEGIN IMMEDIATE")
-                try:
-                    # Conditional on the row still being what was read, so a
-                    # concurrent write is not clobbered.
-                    changed = 0
-                    for new_type, rowid in moves:
-                        cur = active.execute(
-                            f"UPDATE {_TABLE} SET fact_type = ? "
-                            f"WHERE rowid = ? AND fact_type = '{_PROSPECTIVE}'",
-                            (new_type, rowid),
-                        )
-                        # Count what the guard let through, not what was
-                        # offered. A concurrent write can change the row
-                        # underneath, and a log line that reports the intention
-                        # as the outcome is how a receipt comes to overstate
-                        # what happened.
-                        changed += cur.rowcount if cur.rowcount and cur.rowcount > 0 else 0
-                    active.commit()
-                except Exception:
-                    active.rollback()
-                    raise
-                demoted += changed
-
-    logger.info(
-        "M048: re-read %d memories filed as plans; %d were, %d moved",
-        demoted + kept, kept, demoted,
-    )
-
-
-def verify(conn: sqlite3.Connection) -> bool:
-    """Is the end state in place: would running this pass again change nothing?
-
-    That is the question the runner asks a verify, and it is answerable exactly
-    — this pass is a pure function of the text, so its end state is its own
-    fixed point. No estimate, no threshold.
-
-    An earlier version tried to tell "the pass never ran" from "the rule got
-    sharper" by how much disagreed, and a share cannot carry that: a store with
-    nine memories, six of them real plans, sits at 33% disagreement having never
-    been touched at all. It reported success.
-
-    Drift after a rule change makes this answer False, correctly — the end state
-    under today's rule is genuinely not in place. It is not left to the runner
-    to repair, because a completed migration is never replayed: the maintenance
-    cycle re-runs the same pass, so the store converges on its own. See
-    ``core/maintenance_scheduler``.
-    """
-    existing = {r[1] for r in conn.execute(f"PRAGMA table_info({_TABLE})")}
-    if "fact_type" not in existing or "content" not in existing:
-        return True
-
-    rows = conn.execute(
-        f"SELECT fact_id, content FROM {_TABLE} WHERE fact_type = ?",
-        (_PROSPECTIVE,),
-    ).fetchall()
-    if not rows:
-        return True
-
-    drifted = [
-        fact_id for fact_id, content in rows if _resolve(content) != _PROSPECTIVE
-    ]
-    if drifted:
-        logger.info(
-            "M048: %d of %d memories filed as plans no longer read as one; the "
-            "maintenance cycle re-reads them", len(drifted), len(rows),
-        )
-        return False
-    return True
-
-
-def blocks_serving(conn: sqlite3.Connection) -> bool:
-    """Should a daemon refuse to serve while this check does not hold? No.
-
-    Nothing here is about schema. ``verify()`` reads ``content`` and compares
-    today's reading of it against the ``fact_type`` already stored, so a False
-    means some memories are filed as plans that no longer read as plans. Every
-    table and column a query needs is present either way; the store answers
-    normally, and at worst a handful of memories carry a stale label until the
-    next maintenance pass re-reads them.
-
-    The distinction matters because this is a standing guard over data that
-    ordinary use re-violates by design: a plan whose date passes stops reading
-    as upcoming, which is the rule working, not a fault. Treating that like a
-    missing table let one drifted row answer 503 on every route for as long as
-    the process lived — and because the readiness snapshot is taken once at
-    startup, the background pass that repairs the data could not lift the
-    refusal it caused. An outage produced by a quality check is worse than the
-    thing the check is for. Same reasoning as ``M043.blocks_serving``.
-    """
-    return False