superlocalmemory 4.1.13 → 4.1.15

package/src/superlocalmemory/storage/migrations/M017_ccq_scope_column.py

@@ -1,79 +0,0 @@
-# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
-# Licensed under AGPL-3.0-or-later - see LICENSE file
-# Part of SuperLocalMemory v3.6.18
-
-"""M017 — scope column on ccq_consolidated_blocks.
-
-The cognitive-consolidation queue table (``ccq_consolidated_blocks``) was
-created before the M016 multi-scope migration and therefore has no ``scope``
-column.  Without it, consolidation summaries are always stored as
-``personal`` regardless of the source facts' scope — a silent data-loss edge
-case when a user has opted into global or shared memory.
-
-Fix: add a ``scope TEXT NOT NULL DEFAULT 'personal'`` column plus a
-``(profile_id, scope)`` covering index for scope-filtered queries.
-
-Deferred (like M006, M011, M013, M016): the CCQ table is created by the
-engine, not by migration DDL, so we apply after engine init via
-``apply_deferred``.
-
-Author: Varun Pratap Bhardwaj / Qualixar
-"""
-
-from __future__ import annotations
-
-import sqlite3
-
-NAME = "M017_ccq_scope_column"
-DB_TARGET = "memory"
-
-TABLE = "ccq_consolidated_blocks"
-
-DDL = (
-    f"ALTER TABLE {TABLE} ADD COLUMN scope TEXT NOT NULL DEFAULT 'personal';"
-    f"CREATE INDEX IF NOT EXISTS idx_{TABLE}_scope ON {TABLE}(scope);"
-    f"CREATE INDEX IF NOT EXISTS idx_{TABLE}_profile_scope ON {TABLE}(profile_id, scope);"
-)
-
-
-def _table_exists(conn: sqlite3.Connection, table: str) -> bool:
-    return conn.execute(
-        "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (table,)
-    ).fetchone() is not None
-
-
-def _column_names(conn: sqlite3.Connection, table: str) -> set[str]:
-    return {r[1] for r in conn.execute(f"PRAGMA table_info({table})").fetchall()}
-
-
-def apply(conn: sqlite3.Connection) -> None:
-    """Idempotently add scope column + indexes to ccq_consolidated_blocks.
-
-    Skips silently when the table doesn't exist yet (engine hasn't created it)
-    or when the column is already present (re-apply or fresh install).
-    """
-    if not _table_exists(conn, TABLE):
-        return
-    cols = _column_names(conn, TABLE)
-    if "scope" not in cols:
-        conn.execute(
-            f"ALTER TABLE {TABLE} ADD COLUMN scope TEXT NOT NULL DEFAULT 'personal'"
-        )
-    conn.execute(f"CREATE INDEX IF NOT EXISTS idx_{TABLE}_scope ON {TABLE}(scope)")
-    conn.execute(
-        f"CREATE INDEX IF NOT EXISTS idx_{TABLE}_profile_scope ON {TABLE}(profile_id, scope)"
-    )
-
-
-def verify(conn: sqlite3.Connection) -> bool:
-    """Applied when ccq_consolidated_blocks has the scope column and index."""
-    if not _table_exists(conn, TABLE):
-        return True  # table absent — apply() skips it; nothing to verify
-    cols = _column_names(conn, TABLE)
-    if "scope" not in cols:
-        return False
-    idx = conn.execute(
-        "SELECT 1 FROM sqlite_master WHERE type='index' AND name=?",
-        (f"idx_{TABLE}_scope",),
-    ).fetchone()
-    return idx is not None

package/src/superlocalmemory/storage/migrations/M018_ingestion_operations.py

@@ -1,120 +0,0 @@
-# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
-# Licensed under AGPL-3.0-or-later - see LICENSE file
-# Part of SuperLocalMemory V3
-
-"""M018 — durable canonical-ingestion operation records.
-
-This is the EXPAND phase of the V3.7 ingestion migration.  It is additive:
-legacy ``pending_memories`` and ``ingestion_log`` remain untouched until the
-backfill and dual-write comparison prove the new contract.
-"""
-
-from __future__ import annotations
-
-import sqlite3
-
-NAME = "M018_ingestion_operations"
-DB_TARGET = "memory"
-
-DDL = """
-CREATE TABLE IF NOT EXISTS ingestion_operations (
-    operation_id TEXT PRIMARY KEY,
-    profile_id TEXT NOT NULL,
-    source_type TEXT NOT NULL,
-    idempotency_key TEXT NOT NULL,
-    source_hash TEXT NOT NULL,
-    raw_content TEXT NOT NULL,
-    raw_metadata_json TEXT NOT NULL DEFAULT '{}',
-    scope TEXT NOT NULL DEFAULT 'personal'
-        CHECK (scope IN ('personal', 'project', 'shared', 'global')),
-    shared_with_json TEXT NOT NULL DEFAULT '[]',
-    trusted_actor_id TEXT NOT NULL DEFAULT '',
-    session_id TEXT NOT NULL DEFAULT '',
-    session_date TEXT NOT NULL DEFAULT '',
-    speaker TEXT NOT NULL DEFAULT '',
-    role TEXT NOT NULL DEFAULT 'user',
-    state TEXT NOT NULL DEFAULT 'raw'
-        CHECK (state IN ('raw', 'queryable', 'enriching', 'complete', 'failed')),
-    queryable_fact_ids_json TEXT NOT NULL DEFAULT '[]',
-    final_fact_ids_json TEXT NOT NULL DEFAULT '[]',
-    derivation_version TEXT NOT NULL DEFAULT '',
-    derivation_state_json TEXT NOT NULL DEFAULT '{}',
-    lease_owner TEXT NOT NULL DEFAULT '',
-    lease_expires_at REAL NOT NULL DEFAULT 0,
-    next_retry_at REAL NOT NULL DEFAULT 0,
-    attempt_count INTEGER NOT NULL DEFAULT 0 CHECK (attempt_count >= 0),
-    last_error TEXT NOT NULL DEFAULT '',
-    created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
-    updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
-    UNIQUE(profile_id, source_type, idempotency_key)
-);
-CREATE INDEX IF NOT EXISTS idx_ingestion_operations_state
-    ON ingestion_operations(state, updated_at);
-CREATE INDEX IF NOT EXISTS idx_ingestion_operations_source_hash
-    ON ingestion_operations(profile_id, source_hash);
-"""
-
-
-def apply(conn: sqlite3.Connection) -> None:
-    """Create the additive operation table and indexes idempotently."""
-    conn.executescript(DDL)
-    columns = {
-        row[1]
-        for row in conn.execute("PRAGMA table_info(ingestion_operations)").fetchall()
-    }
-    additive_columns = {
-        "derivation_state_json": "TEXT NOT NULL DEFAULT '{}'",
-        "session_date": "TEXT NOT NULL DEFAULT ''",
-        "speaker": "TEXT NOT NULL DEFAULT ''",
-        "role": "TEXT NOT NULL DEFAULT 'user'",
-        "lease_owner": "TEXT NOT NULL DEFAULT ''",
-        "lease_expires_at": "REAL NOT NULL DEFAULT 0",
-        "next_retry_at": "REAL NOT NULL DEFAULT 0",
-    }
-    for name, declaration in additive_columns.items():
-        if name not in columns:
-            conn.execute(
-                f"ALTER TABLE ingestion_operations ADD COLUMN {name} {declaration}"
-            )
-
-
-def verify(conn: sqlite3.Connection) -> bool:
-    """Return true only when the complete M018 contract is present."""
-    table = conn.execute(
-        "SELECT 1 FROM sqlite_master WHERE type='table' "
-        "AND name='ingestion_operations'"
-    ).fetchone()
-    if table is None:
-        return False
-    columns = {
-        row[1]
-        for row in conn.execute("PRAGMA table_info(ingestion_operations)").fetchall()
-    }
-    required = {
-        "operation_id",
-        "profile_id",
-        "source_type",
-        "idempotency_key",
-        "source_hash",
-        "raw_content",
-        "state",
-        "queryable_fact_ids_json",
-        "final_fact_ids_json",
-        "derivation_version",
-        "derivation_state_json",
-        "session_date",
-        "speaker",
-        "role",
-        "lease_owner",
-        "lease_expires_at",
-        "next_retry_at",
-        "attempt_count",
-        "last_error",
-    }
-    if not required <= columns:
-        return False
-    indexes = {
-        row[1]
-        for row in conn.execute("PRAGMA index_list(ingestion_operations)").fetchall()
-    }
-    return "idx_ingestion_operations_state" in indexes

package/src/superlocalmemory/storage/migrations/M019_derivation_lineage.py

@@ -1,54 +0,0 @@
-# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
-# Licensed under AGPL-3.0-or-later - see LICENSE file
-
-"""M019 — durable, non-fabricated derivation lineage."""
-
-from __future__ import annotations
-
-import sqlite3
-
-NAME = "M019_derivation_lineage"
-DB_TARGET = "memory"
-
-DDL = """
-CREATE TABLE IF NOT EXISTS derivation_lineage (
-    lineage_id TEXT PRIMARY KEY,
-    profile_id TEXT NOT NULL,
-    object_type TEXT NOT NULL,
-    object_id TEXT NOT NULL,
-    operation_id TEXT NOT NULL DEFAULT '',
-    derivation_version TEXT NOT NULL DEFAULT '',
-    source_status TEXT NOT NULL
-        CHECK (source_status IN ('exact', 'derived_from_facts', 'unresolved', 'not_applicable')),
-    source_start INTEGER,
-    source_end INTEGER,
-    source_text_sha256 TEXT NOT NULL DEFAULT '',
-    source_fact_ids_json TEXT NOT NULL DEFAULT '[]',
-    unresolved_reason TEXT NOT NULL DEFAULT '',
-    created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
-    UNIQUE(profile_id, object_type, object_id, operation_id)
-);
-CREATE INDEX IF NOT EXISTS idx_derivation_lineage_profile
-    ON derivation_lineage(profile_id, object_type, object_id);
-CREATE INDEX IF NOT EXISTS idx_derivation_lineage_operation
-    ON derivation_lineage(operation_id);
-"""
-
-
-def apply(conn: sqlite3.Connection) -> None:
-    conn.executescript(DDL)
-
-
-def verify(conn: sqlite3.Connection) -> bool:
-    table = conn.execute(
-        "SELECT 1 FROM sqlite_master WHERE type='table' AND name='derivation_lineage'"
-    ).fetchone()
-    if table is None:
-        return False
-    columns = {row[1] for row in conn.execute("PRAGMA table_info(derivation_lineage)")}
-    return {
-        "lineage_id", "profile_id", "object_type", "object_id",
-        "operation_id", "derivation_version", "source_status",
-        "source_start", "source_end", "source_text_sha256",
-        "source_fact_ids_json", "unresolved_reason",
-    } <= columns

package/src/superlocalmemory/storage/migrations/M020_model_state_integrity.py

@@ -1,52 +0,0 @@
-# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
-# Licensed under AGPL-3.0-or-later - see LICENSE file
-
-"""M020 — forward-only integrity repair for learned model state.
-
-M002's DDL shipped before model-state SHA values were backfilled.  Do not
-modify M002: its stored hash is part of every existing user's upgrade record.
-This migration deterministically fills only missing digests and never changes
-the model payload or ranking metadata.
-"""
-
-from __future__ import annotations
-
-import hashlib
-import sqlite3
-
-NAME = "M020_model_state_integrity"
-DB_TARGET = "learning"
-
-# The runner records a stable fingerprint even though this migration uses a
-# Python apply function for binary-safe SHA computation.
-DDL = "-- M020: backfill missing learning_model_state.bytes_sha256\n"
-
-
-def verify(conn: sqlite3.Connection) -> bool:
-    """Return True only when every persisted model has an integrity digest."""
-    try:
-        missing = conn.execute(
-            "SELECT 1 FROM learning_model_state "
-            "WHERE bytes_sha256 = '' OR bytes_sha256 IS NULL LIMIT 1"
-        ).fetchone()
-    except sqlite3.Error:
-        return False
-    return missing is None
-
-
-def apply(conn: sqlite3.Connection) -> None:
-    """Backfill SHA-256 values without modifying model bytes or metadata."""
-    rows = conn.execute(
-        "SELECT id, state_bytes FROM learning_model_state "
-        "WHERE bytes_sha256 = '' OR bytes_sha256 IS NULL"
-    ).fetchall()
-    updates = [
-        (hashlib.sha256(state_bytes).hexdigest(), row_id)
-        for row_id, state_bytes in rows
-        if state_bytes is not None
-    ]
-    if updates:
-        conn.executemany(
-            "UPDATE learning_model_state SET bytes_sha256 = ? WHERE id = ?",
-            updates,
-        )

package/src/superlocalmemory/storage/migrations/M021_ingestion_log_profile.py

@@ -1,108 +0,0 @@
-# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
-# Licensed under AGPL-3.0-or-later - see LICENSE file
-# Part of SuperLocalMemory — per-profile isolation (I-4)
-
-"""M021 — per-profile ingestion_log dedup (memory.db, deferred).
-
-``ingestion_log`` is the compatibility ledger for external-adapter ingests. Its
-dedup UNIQUE constraint was global — ``UNIQUE(source_type, dedup_key)`` — so a
-second profile ingesting the same source key could not record its own ledger
-row (INSERT OR IGNORE silently no-op'd), losing that profile's fact_ids in the
-ledger. (The authoritative dedup already moved to the M018 ingestion_operations
-table, which is profile-scoped, so the memory itself is not starved — this
-migration fixes the ledger so per-profile bookkeeping is complete.)
-
-SQLite cannot alter a UNIQUE constraint in place, so the table is rebuilt:
-rename → create with profile_id + UNIQUE(profile_id, source_type, dedup_key) →
-copy rows backfilling profile_id='default' → drop old. Idempotent and tolerant
-of a missing table (fresh installs create it correctly via schema_v343).
-
-Deferred because ingestion_log is created at engine init (apply_v343_schema),
-after ``apply_all`` runs — same reason as M016.
-
-Author: Varun Pratap Bhardwaj / Qualixar
-"""
-
-from __future__ import annotations
-
-import sqlite3
-
-NAME = "M021_ingestion_log_profile"
-DB_TARGET = "memory"
-
-# Documentation + drift hash. apply() below is the authoritative executor.
-DDL = (
-    "ALTER TABLE ingestion_log ADD COLUMN profile_id TEXT NOT NULL DEFAULT 'default';"
-    "-- rebuild for UNIQUE(profile_id, source_type, dedup_key)"
-)
-
-_NEW_TABLE = """
-CREATE TABLE ingestion_log (
-    id INTEGER PRIMARY KEY AUTOINCREMENT,
-    profile_id TEXT NOT NULL DEFAULT 'default',
-    source_type TEXT NOT NULL,
-    dedup_key TEXT NOT NULL,
-    fact_ids TEXT DEFAULT '[]',
-    metadata TEXT DEFAULT '{}',
-    status TEXT DEFAULT 'ingested',
-    ingested_at TEXT NOT NULL,
-    UNIQUE(profile_id, source_type, dedup_key)
-)
-"""
-
-
-def _cols(conn: sqlite3.Connection, table: str) -> set[str]:
-    return {r[1] for r in conn.execute(f"PRAGMA table_info({table})").fetchall()}
-
-
-def _table_exists(conn: sqlite3.Connection, table: str) -> bool:
-    return conn.execute(
-        "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (table,)
-    ).fetchone() is not None
-
-
-def apply(conn: sqlite3.Connection) -> None:
-    """Rebuild ingestion_log with a profile-scoped dedup constraint.
-
-    No-op on a fresh install (table already has profile_id) or when the table
-    doesn't exist yet. Existing rows backfill to the 'default' profile.
-    """
-    if not _table_exists(conn, "ingestion_log"):
-        return
-    if "profile_id" in _cols(conn, "ingestion_log"):
-        return
-
-    conn.execute("ALTER TABLE ingestion_log RENAME TO _ingestion_log_old")
-    conn.executescript(_NEW_TABLE)
-    # Copy legacy rows under the 'default' profile. Column set matches the
-    # pre-migration schema (id, source_type, dedup_key, fact_ids, metadata,
-    # status, ingested_at).
-    old_cols = _cols(conn, "_ingestion_log_old")
-    has_meta = "metadata" in old_cols
-    if has_meta:
-        conn.execute(
-            "INSERT INTO ingestion_log "
-            "(id, profile_id, source_type, dedup_key, fact_ids, metadata, "
-            " status, ingested_at) "
-            "SELECT id, 'default', source_type, dedup_key, fact_ids, metadata, "
-            " status, ingested_at FROM _ingestion_log_old"
-        )
-    else:
-        conn.execute(
-            "INSERT INTO ingestion_log "
-            "(id, profile_id, source_type, dedup_key, fact_ids, status, ingested_at) "
-            "SELECT id, 'default', source_type, dedup_key, fact_ids, "
-            " status, ingested_at FROM _ingestion_log_old"
-        )
-    conn.execute("DROP TABLE _ingestion_log_old")
-    conn.execute(
-        "CREATE INDEX IF NOT EXISTS idx_ingestion_dedup "
-        "ON ingestion_log(profile_id, source_type, dedup_key)"
-    )
-
-
-def verify(conn: sqlite3.Connection) -> bool:
-    """Applied once ingestion_log carries profile_id (or is absent on fresh DB)."""
-    if not _table_exists(conn, "ingestion_log"):
-        return True  # nothing to migrate; fresh install creates it correctly
-    return "profile_id" in _cols(conn, "ingestion_log")

package/src/superlocalmemory/storage/migrations/M022_entity_aliases_profile.py

@@ -1,86 +0,0 @@
-# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
-# Licensed under AGPL-3.0-or-later - see LICENSE file
-# Part of SuperLocalMemory — per-profile isolation (I-7)
-
-"""M022 — profile_id on entity_aliases (memory.db, deferred).
-
-entity_aliases had no profile_id, so aliases were keyed by entity_id alone. When
-the same entity_id appears under two profiles, one profile's aliases were
-visible to (and deduped against) the other. Add profile_id and backfill each
-alias from its parent canonical entity's profile (the correct owner), falling
-back to 'default' for orphans.
-
-Additive column — no UNIQUE change, so a simple ADD COLUMN + backfill. Deferred
-because entity_aliases is created at engine init (create_all_tables), after
-apply_all runs — same reason as M016.
-
-Author: Varun Pratap Bhardwaj / Qualixar
-"""
-
-from __future__ import annotations
-
-import sqlite3
-
-NAME = "M022_entity_aliases_profile"
-DB_TARGET = "memory"
-
-DDL = "ALTER TABLE entity_aliases ADD COLUMN profile_id TEXT NOT NULL DEFAULT 'default'"
-
-
-def _cols(conn: sqlite3.Connection, table: str) -> set[str]:
-    return {r[1] for r in conn.execute(f"PRAGMA table_info({table})").fetchall()}
-
-
-def _table_exists(conn: sqlite3.Connection, table: str) -> bool:
-    return conn.execute(
-        "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (table,)
-    ).fetchone() is not None
-
-
-def apply(conn: sqlite3.Connection) -> None:
-    """Add profile_id to entity_aliases and backfill from the parent entity.
-
-    No-op on a fresh install (column already present) or a missing table.
-    """
-    if not _table_exists(conn, "entity_aliases"):
-        return
-
-    if "profile_id" not in _cols(conn, "entity_aliases"):
-        conn.execute(
-            "ALTER TABLE entity_aliases "
-            "ADD COLUMN profile_id TEXT NOT NULL DEFAULT 'default'"
-        )
-        # Backfill each alias from its parent canonical entity's profile. If the
-        # parent is missing (orphan) the COALESCE keeps 'default'.
-        if _table_exists(conn, "canonical_entities"):
-            conn.execute(
-                "UPDATE entity_aliases SET profile_id = COALESCE("
-                " (SELECT ce.profile_id FROM canonical_entities ce "
-                "  WHERE ce.entity_id = entity_aliases.entity_id LIMIT 1), 'default')"
-            )
-
-    # Always ensure the index (this migration owns it — schema.create_all_tables
-    # deliberately does NOT create it, since on an upgrade the column doesn't
-    # exist yet at create-all time). Idempotent for fresh installs too.
-    conn.execute(
-        "CREATE INDEX IF NOT EXISTS idx_aliases_profile_entity "
-        "ON entity_aliases(profile_id, entity_id)"
-    )
-
-
-def verify(conn: sqlite3.Connection) -> bool:
-    """Applied once entity_aliases has profile_id AND its index (or is absent).
-
-    Checking the index (not just the column) means a fresh install — where
-    create_all_tables made the column but not the index — still runs apply()
-    to create the index.
-    """
-    if not _table_exists(conn, "entity_aliases"):
-        return True
-    if "profile_id" not in _cols(conn, "entity_aliases"):
-        return False
-    idx = conn.execute(
-        "SELECT 1 FROM sqlite_master WHERE type='index' "
-        "AND name='idx_aliases_profile_entity'"
-    ).fetchone()
-    return idx is not None