superlocalmemory 4.1.13 → 4.1.14

package/src/superlocalmemory/storage/migrations/M028_fact_entity_associations.py

@@ -1,305 +0,0 @@
-# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
-# Licensed under AGPL-3.0-or-later
-
-"""M028 — normalized fact/entity associations for O(1) ingest effects.
-
-The table is both an association index and an idempotency ledger.  Its
-composite primary key means a fact can contribute to an entity's ``fact_count``
-once, regardless of ingestion retries or how many operations converge on the
-same consolidated fact.
-"""
-
-from __future__ import annotations
-
-import json
-import sqlite3
-from datetime import UTC, datetime
-from pathlib import Path
-
-NAME = "M028_fact_entity_associations"
-DB_TARGET = "memory"
-
-DDL = """
-CREATE TABLE IF NOT EXISTS fact_entity_associations (
-    profile_id TEXT NOT NULL,
-    fact_id TEXT NOT NULL,
-    entity_id TEXT NOT NULL,
-    first_operation_id TEXT NOT NULL DEFAULT '',
-    count_applied INTEGER NOT NULL DEFAULT 0
-        CHECK (count_applied IN (0, 1)),
-    created_at TEXT NOT NULL DEFAULT (
-        strftime('%Y-%m-%dT%H:%M:%fZ', 'now')
-    ),
-    PRIMARY KEY (profile_id, fact_id, entity_id),
-    FOREIGN KEY (fact_id) REFERENCES atomic_facts(fact_id) ON DELETE CASCADE,
-    FOREIGN KEY (entity_id)
-        REFERENCES canonical_entities(entity_id) ON DELETE CASCADE
-);
-CREATE INDEX IF NOT EXISTS idx_fact_entity_associations_entity
-    ON fact_entity_associations(profile_id, entity_id, fact_id);
-CREATE TABLE IF NOT EXISTS fact_entity_association_repair_state (
-    repair_key TEXT PRIMARY KEY,
-    state TEXT NOT NULL DEFAULT 'pending'
-        CHECK (state IN ('pending', 'running', 'retrying', 'complete')),
-    target_fact_rowid INTEGER NOT NULL DEFAULT -1,
-    last_fact_rowid INTEGER NOT NULL DEFAULT 0,
-    scanned INTEGER NOT NULL DEFAULT 0,
-    inserted INTEGER NOT NULL DEFAULT 0,
-    last_error TEXT NOT NULL DEFAULT '',
-    updated_at TEXT NOT NULL
-);
-"""
-
-
-def _table_exists(conn: sqlite3.Connection, table: str) -> bool:
-    return conn.execute(
-        "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?",
-        (table,),
-    ).fetchone() is not None
-
-
-def apply(conn: sqlite3.Connection) -> None:
-    """Install schema and capture a constant-time historical rowid boundary."""
-    conn.executescript(DDL)
-    association_columns = {
-        row[1]
-        for row in conn.execute(
-            "PRAGMA table_info(fact_entity_associations)"
-        ).fetchall()
-    }
-    if "count_applied" not in association_columns:
-        conn.execute(
-            "ALTER TABLE fact_entity_associations "
-            "ADD COLUMN count_applied INTEGER NOT NULL DEFAULT 0 "
-            "CHECK (count_applied IN (0, 1))"
-        )
-    repair_columns = {
-        row[1]
-        for row in conn.execute(
-            "PRAGMA table_info(fact_entity_association_repair_state)"
-        ).fetchall()
-    }
-    if "target_fact_rowid" not in repair_columns:
-        conn.execute(
-            "ALTER TABLE fact_entity_association_repair_state "
-            "ADD COLUMN target_fact_rowid INTEGER NOT NULL DEFAULT -1"
-        )
-    target = int(conn.execute(
-        "SELECT COALESCE(MAX(rowid), 0) FROM atomic_facts"
-    ).fetchone()[0])
-    conn.executescript(
-        "INSERT OR IGNORE INTO fact_entity_association_repair_state "
-        "(repair_key,state,target_fact_rowid,updated_at) "
-        f"VALUES ('historical-backfill','pending',{target},'');"
-        "UPDATE fact_entity_association_repair_state "
-        f"SET target_fact_rowid={target} "
-        "WHERE repair_key='historical-backfill' AND target_fact_rowid < 0;"
-    )
-
-
-def _now() -> str:
-    return datetime.now(UTC).isoformat()
-
-
-def _connect_read(db_path: Path) -> sqlite3.Connection:
-    """Read-only connection with system-default busy_timeout.
-
-    Concurrency fix (v3.8.4): busy_timeout raised from 5 s to match the
-    system default (10 s, env SLM_DB_BUSY_TIMEOUT_MS).  Timeout raised from
-    5 s to 10 s for the same reason.  Write connections now go through
-    memory_write() so they acquire get_write_lock() and never use this helper.
-    """
-    import os
-
-    try:
-        ms = max(0, int(os.environ.get("SLM_DB_BUSY_TIMEOUT_MS", "10000")))
-    except (TypeError, ValueError):
-        ms = 10000
-    conn = sqlite3.connect(str(db_path), timeout=ms / 1000.0)
-    conn.row_factory = sqlite3.Row
-    conn.execute(f"PRAGMA busy_timeout={ms}")
-    conn.execute("PRAGMA foreign_keys=ON")
-    return conn
-
-
-def get_repair_status(db_path: Path) -> dict[str, int | str]:
-    """Read durable backfill progress without inferring it from schema."""
-    conn = _connect_read(Path(db_path))
-    try:
-        row = conn.execute(
-            "SELECT state,target_fact_rowid,last_fact_rowid,scanned,inserted,"
-            "last_error,updated_at "
-            "FROM fact_entity_association_repair_state "
-            "WHERE repair_key='historical-backfill'"
-        ).fetchone()
-        if row is None:
-            return {
-                "state": "pending", "target_fact_rowid": 0,
-                "last_fact_rowid": 0,
-                "scanned": 0, "inserted": 0,
-                "last_error": "", "updated_at": "",
-            }
-        return dict(row)
-    finally:
-        conn.close()
-
-
-def _entity_ids(raw: object) -> tuple[str, ...]:
-    try:
-        values = json.loads(str(raw or "[]"))
-    except (TypeError, ValueError):
-        return ()
-    if not isinstance(values, list):
-        return ()
-    return tuple(dict.fromkeys(str(value) for value in values if value))
-
-
-def _repair_batch(conn: sqlite3.Connection, batch_size: int) -> dict[str, int | bool]:
-    """Execute one backfill batch on *conn*.
-
-    Concurrency fix (v3.8.4): caller MUST hold the write lock via
-    memory_write() before calling this function.  The explicit
-    ``conn.execute("BEGIN IMMEDIATE")`` has been removed because:
-
-    * memory_write() already acquired get_write_lock() (Python-level
-      serialisation) — no other in-process writer can start.
-    * SQLite's implicit deferred transaction is promoted to a write
-      transaction on the first INSERT, which is equivalent to BEGIN
-      IMMEDIATE for in-process serialisation.
-    * The explicit BEGIN IMMEDIATE previously bypassed get_write_lock()
-      and would retry at the SQLite layer only (busy_timeout=5 s, now
-      10 s) without respecting the Python-level write lock order.
-
-    Transaction boundary (commit/rollback) is managed by memory_write().
-    """
-    status = conn.execute(
-        "SELECT last_fact_rowid,target_fact_rowid "
-        "FROM fact_entity_association_repair_state "
-        "WHERE repair_key='historical-backfill'"
-    ).fetchone()
-    if status is None:
-        return {"scanned": 0, "inserted": 0, "complete": True}
-    cursor = int(status["last_fact_rowid"] or 0)
-    target = int(status["target_fact_rowid"])
-    rows = conn.execute(
-        "SELECT rowid,fact_id,profile_id,canonical_entities_json "
-        "FROM atomic_facts WHERE rowid>? AND rowid<=? "
-        "ORDER BY rowid LIMIT ?",
-        (cursor, target, batch_size),
-    ).fetchall()
-    if not rows:
-        conn.execute(
-            "UPDATE fact_entity_association_repair_state "
-            "SET state='complete',last_error='',updated_at=? "
-            "WHERE repair_key='historical-backfill'",
-            (_now(),),
-        )
-        return {"scanned": 0, "inserted": 0, "complete": True}
-    inserted = 0
-    for row in rows:
-        for entity_id in _entity_ids(row["canonical_entities_json"]):
-            result = conn.execute(
-                "INSERT OR IGNORE INTO fact_entity_associations "
-                "(profile_id,fact_id,entity_id,first_operation_id,"
-                "count_applied) "
-                "SELECT ?,?,?,?,? FROM canonical_entities "
-                "WHERE profile_id=? AND entity_id=?",
-                (
-                    row["profile_id"], row["fact_id"], entity_id,
-                    "migration-backfill", 0,
-                    row["profile_id"], entity_id,
-                ),
-            )
-            inserted += max(0, result.rowcount)
-    conn.execute(
-        "UPDATE fact_entity_association_repair_state SET "
-        "state='running',last_fact_rowid=?,scanned=scanned+?,"
-        "inserted=inserted+?,last_error='',updated_at=? "
-        "WHERE repair_key='historical-backfill'",
-        (int(rows[-1]["rowid"]), len(rows), inserted, _now()),
-    )
-    return {"scanned": len(rows), "inserted": inserted, "complete": False}
-
-
-def repair_fact_entity_associations(
-    db_path: Path,
-    *,
-    batch_size: int = 250,
-    max_batches: int = 1,
-) -> dict[str, int | bool]:
-    """Run bounded, restartable short-transaction backfill batches.
-
-    Concurrency fix (v3.8.4): each batch is now wrapped in memory_write()
-    which acquires get_write_lock() (process-level write serialisation) and
-    sets the system-default busy_timeout (10 s) before opening the SQLite
-    connection.  Previously the loop used a single raw _connect() (timeout=5,
-    busy_timeout=5000) without get_write_lock(), which bypassed in-process
-    write serialisation and could cause SQLITE_BUSY after only 5 s.
-    """
-    from superlocalmemory.storage.memory_write import memory_write
-
-    if batch_size < 1 or max_batches < 1:
-        raise ValueError("batch_size and max_batches must be positive")
-    totals: dict[str, int | bool] = {"scanned": 0, "inserted": 0, "complete": False}
-    for _ in range(max_batches):
-        try:
-            with memory_write(Path(db_path)) as conn:
-                conn.execute("PRAGMA foreign_keys=ON")
-                result = _repair_batch(conn, batch_size)
-                totals["scanned"] += int(result["scanned"])
-                totals["inserted"] += int(result["inserted"])
-                totals["complete"] = bool(result["complete"])
-        except sqlite3.Error as exc:
-            # On error, record the retrying state in a separate short write.
-            try:
-                with memory_write(Path(db_path)) as econn:
-                    econn.execute(
-                        "UPDATE fact_entity_association_repair_state SET "
-                        "state='retrying',last_error=?,updated_at=? "
-                        "WHERE repair_key='historical-backfill'",
-                        (type(exc).__name__, _now()),
-                    )
-            except sqlite3.Error:
-                pass
-            raise
-        if totals["complete"]:
-            break
-    return totals
-
-
-def verify(conn: sqlite3.Connection) -> bool:
-    """Return true when the normalized association contract is indexed."""
-    if not (
-        _table_exists(conn, "fact_entity_associations")
-        and _table_exists(conn, "fact_entity_association_repair_state")
-    ):
-        return False
-    association_columns = {
-        row[1]
-        for row in conn.execute(
-            "PRAGMA table_info(fact_entity_associations)"
-        ).fetchall()
-    }
-    repair_columns = {
-        row[1]
-        for row in conn.execute(
-            "PRAGMA table_info(fact_entity_association_repair_state)"
-        ).fetchall()
-    }
-    if (
-        "count_applied" not in association_columns
-        or "target_fact_rowid" not in repair_columns
-    ):
-        return False
-    indexes = {
-        row[1]
-        for row in conn.execute(
-            "PRAGMA index_list(fact_entity_associations)"
-        ).fetchall()
-    }
-    return "idx_fact_entity_associations_entity" in indexes
-
-
-def repair(conn: sqlite3.Connection) -> None:
-    """Restore additive M028 schema without recapturing its high-water mark."""
-    apply(conn)

package/src/superlocalmemory/storage/migrations/M029_behavioral_history_indexes.py

@@ -1,137 +0,0 @@
-# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
-# Licensed under AGPL-3.0-or-later
-
-"""M029 — profile-scoped indexes for large behavioral histories.
-
-Dashboard reads always filter behavioral and outcome history by profile before
-sorting or aggregating. These composite indexes keep those operations bounded
-to one tenant as the shared database grows.
-"""
-
-from __future__ import annotations
-
-import sqlite3
-
-NAME = "M029_behavioral_history_indexes"
-DB_TARGET = "memory"
-
-DDL = """
-CREATE INDEX IF NOT EXISTS idx_outcomes_profile_outcome_time
-    ON action_outcomes(profile_id, outcome, timestamp DESC);
-CREATE INDEX IF NOT EXISTS idx_outcomes_profile_settled_time
-    ON action_outcomes(profile_id, settled, settled_at DESC);
-CREATE INDEX IF NOT EXISTS idx_outcomes_profile_settled_cursor
-    ON action_outcomes(
-        profile_id, settled, COALESCE(settled_at, ''), outcome_id
-    );
-CREATE INDEX IF NOT EXISTS idx_assertions_profile_confidence
-    ON behavioral_assertions(profile_id, confidence DESC);
-CREATE INDEX IF NOT EXISTS idx_tool_events_profile_created
-    ON tool_events(profile_id, created_at DESC);
-CREATE INDEX IF NOT EXISTS idx_soft_prompts_profile_active_category
-    ON soft_prompt_templates(profile_id, active, category, prompt_id);
-"""
-
-_INDEXES: tuple[tuple[str, str, tuple[str, ...]], ...] = (
-    (
-        "idx_outcomes_profile_outcome_time",
-        "action_outcomes",
-        ("profile_id", "outcome", "timestamp"),
-    ),
-    (
-        "idx_outcomes_profile_settled_time",
-        "action_outcomes",
-        ("profile_id", "settled", "settled_at"),
-    ),
-    (
-        "idx_outcomes_profile_settled_cursor",
-        "action_outcomes",
-        ("profile_id", "settled", "settled_at", "outcome_id"),
-    ),
-    (
-        "idx_assertions_profile_confidence",
-        "behavioral_assertions",
-        ("profile_id", "confidence"),
-    ),
-    (
-        "idx_tool_events_profile_created",
-        "tool_events",
-        ("profile_id", "created_at"),
-    ),
-    (
-        "idx_soft_prompts_profile_active_category",
-        "soft_prompt_templates",
-        ("profile_id", "active", "category", "prompt_id"),
-    ),
-)
-
-
-def _columns(conn: sqlite3.Connection, table: str) -> set[str]:
-    return {
-        str(row[1])
-        for row in conn.execute(f"PRAGMA table_info({table})").fetchall()
-    }
-
-
-def _index_sql(name: str) -> str:
-    statements = {
-        "idx_outcomes_profile_outcome_time": (
-            "CREATE INDEX IF NOT EXISTS idx_outcomes_profile_outcome_time "
-            "ON action_outcomes(profile_id, outcome, timestamp DESC)"
-        ),
-        "idx_outcomes_profile_settled_time": (
-            "CREATE INDEX IF NOT EXISTS idx_outcomes_profile_settled_time "
-            "ON action_outcomes(profile_id, settled, settled_at DESC)"
-        ),
-        "idx_outcomes_profile_settled_cursor": (
-            "CREATE INDEX IF NOT EXISTS idx_outcomes_profile_settled_cursor "
-            "ON action_outcomes("
-            "profile_id, settled, COALESCE(settled_at, ''), outcome_id)"
-        ),
-        "idx_assertions_profile_confidence": (
-            "CREATE INDEX IF NOT EXISTS idx_assertions_profile_confidence "
-            "ON behavioral_assertions(profile_id, confidence DESC)"
-        ),
-        "idx_tool_events_profile_created": (
-            "CREATE INDEX IF NOT EXISTS idx_tool_events_profile_created "
-            "ON tool_events(profile_id, created_at DESC)"
-        ),
-        "idx_soft_prompts_profile_active_category": (
-            "CREATE INDEX IF NOT EXISTS "
-            "idx_soft_prompts_profile_active_category "
-            "ON soft_prompt_templates(profile_id, active, category, prompt_id)"
-        ),
-    }
-    return statements[name]
-
-
-def apply(conn: sqlite3.Connection) -> None:
-    """Add each index supported by the installed runtime schema.
-
-    Optional v3.2/v3.4.7 tables also declare these indexes in their owning
-    schema modules, so a partial bootstrap cannot permanently miss an index
-    when those tables are created later.
-    """
-    for name, table, required_columns in _INDEXES:
-        if set(required_columns) <= _columns(conn, table):
-            conn.execute(_index_sql(name))
-
-
-def verify(conn: sqlite3.Connection) -> bool:
-    """Require each index whose backing table and columns are present."""
-    names = {
-        str(row[0])
-        for row in conn.execute(
-            "SELECT name FROM sqlite_master WHERE type='index'"
-        ).fetchall()
-    }
-    return all(
-        name in names
-        for name, table, required_columns in _INDEXES
-        if set(required_columns) <= _columns(conn, table)
-    )
-
-
-def repair(conn: sqlite3.Connection) -> None:
-    """Create only indexes supported by the current optional schema."""
-    apply(conn)

package/src/superlocalmemory/storage/migrations/M030_entity_explorer_indexes.py

@@ -1,93 +0,0 @@
-# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
-# Licensed under AGPL-3.0-or-later
-
-"""M030 — page-first Entity Explorer indexes for large installations."""
-
-from __future__ import annotations
-
-import sqlite3
-
-NAME = "M030_entity_explorer_indexes"
-DB_TARGET = "memory"
-
-DDL = """
-CREATE INDEX IF NOT EXISTS idx_entities_profile_fact_count_id
-    ON canonical_entities(profile_id, fact_count DESC, entity_id ASC);
-CREATE INDEX IF NOT EXISTS idx_entities_profile_type_fact_count_id
-    ON canonical_entities(
-        profile_id, entity_type COLLATE NOCASE, fact_count DESC, entity_id ASC
-    );
-CREATE INDEX IF NOT EXISTS idx_entity_profiles_profile_entity_rank
-    ON entity_profiles(
-        profile_id, entity_id, last_compiled_at DESC,
-        project_name COLLATE NOCASE
-    );
-"""
-
-_INDEXES: tuple[tuple[str, str, tuple[str, ...], str], ...] = (
-    (
-        "idx_entities_profile_fact_count_id",
-        "canonical_entities",
-        ("profile_id", "fact_count", "entity_id"),
-        "CREATE INDEX IF NOT EXISTS idx_entities_profile_fact_count_id "
-        "ON canonical_entities(profile_id, fact_count DESC, entity_id ASC)",
-    ),
-    (
-        "idx_entities_profile_type_fact_count_id",
-        "canonical_entities",
-        ("profile_id", "entity_type", "fact_count", "entity_id"),
-        "CREATE INDEX IF NOT EXISTS idx_entities_profile_type_fact_count_id "
-        "ON canonical_entities("
-        "profile_id, entity_type COLLATE NOCASE, fact_count DESC, entity_id ASC"
-        ")",
-    ),
-    (
-        "idx_entity_profiles_profile_entity_rank",
-        "entity_profiles",
-        (
-            "profile_id",
-            "entity_id",
-            "last_compiled_at",
-            "project_name",
-        ),
-        "CREATE INDEX IF NOT EXISTS idx_entity_profiles_profile_entity_rank "
-        "ON entity_profiles("
-        "profile_id, entity_id, last_compiled_at DESC, "
-        "project_name COLLATE NOCASE"
-        ")",
-    ),
-)
-
-
-def _columns(conn: sqlite3.Connection, table: str) -> set[str]:
-    return {
-        str(row[1])
-        for row in conn.execute(f"PRAGMA table_info({table})").fetchall()
-    }
-
-
-def apply(conn: sqlite3.Connection) -> None:
-    """Create every index supported by the installed optional schema."""
-    for _, table, required_columns, sql in _INDEXES:
-        if set(required_columns) <= _columns(conn, table):
-            conn.execute(sql)
-
-
-def verify(conn: sqlite3.Connection) -> bool:
-    """Require indexes only when their complete backing schema exists."""
-    names = {
-        str(row[0])
-        for row in conn.execute(
-            "SELECT name FROM sqlite_master WHERE type='index'"
-        ).fetchall()
-    }
-    return all(
-        name in names
-        for name, table, required_columns, _ in _INDEXES
-        if set(required_columns) <= _columns(conn, table)
-    )
-
-
-def repair(conn: sqlite3.Connection) -> None:
-    """Safely restore a dropped index without replaying data migrations."""
-    apply(conn)

package/src/superlocalmemory/storage/migrations/M031_dead_letter_operations.py

@@ -1,80 +0,0 @@
-# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
-# Licensed under AGPL-3.0-or-later - see LICENSE file
-# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
-
-"""M031 — dead-letter queue for exhausted ingestion operations (Fix E, issue #77).
-
-Additive migration: creates dead_letter_operations if not present.
-Existing rows in ingestion_operations are unaffected.
-
-When an M018 ingestion operation exhausts _MAX_AUTOMATIC_MATERIALIZATION_ATTEMPTS
-(10) it previously remained silently in FAILED state — invisible to operators and
-unreachable by the materialiser.  This table gives operators a persistent,
-inspectable record of every poisoned operation: original content, error, attempt
-count, timestamps, and profile scope.
-
-Schema design:
-  - original_op_id references ingestion_operations.operation_id (soft ref — no FK
-    so that dead-lettered rows survive if the source row is later cleaned up).
-  - profile_id allows per-profile DLQ dashboards.
-  - dead_lettered_at defaults to the current epoch for point-in-time auditing.
-  - No TTL/expiry here — retention policy belongs to a future maintenance sweep.
-"""
-
-from __future__ import annotations
-
-import sqlite3
-
-NAME = "M031_dead_letter_operations"
-DB_TARGET = "memory"
-
-DDL = """
-CREATE TABLE IF NOT EXISTS dead_letter_operations (
-    id                  INTEGER PRIMARY KEY AUTOINCREMENT,
-    original_op_id      TEXT NOT NULL,
-    operation_type      TEXT NOT NULL DEFAULT 'M018',
-    content             TEXT,
-    metadata_json       TEXT,
-    error               TEXT,
-    attempt_count       INTEGER,
-    first_attempt_at    REAL,
-    dead_lettered_at    REAL NOT NULL DEFAULT (unixepoch('now')),
-    profile_id          TEXT
-);
-CREATE INDEX IF NOT EXISTS idx_dlq_profile
-    ON dead_letter_operations (profile_id);
-CREATE INDEX IF NOT EXISTS idx_dlq_op_id
-    ON dead_letter_operations (original_op_id);
-"""
-
-
-def apply(conn: sqlite3.Connection) -> None:
-    """Create the dead_letter_operations table and indexes idempotently."""
-    conn.executescript(DDL)
-
-
-def verify(conn: sqlite3.Connection) -> bool:
-    """Return True only when the complete M031 contract is present."""
-    table = conn.execute(
-        "SELECT 1 FROM sqlite_master WHERE type='table' "
-        "AND name='dead_letter_operations'"
-    ).fetchone()
-    if table is None:
-        return False
-    columns = {
-        row[1]
-        for row in conn.execute(
-            "PRAGMA table_info(dead_letter_operations)"
-        ).fetchall()
-    }
-    required = {
-        "id",
-        "original_op_id",
-        "operation_type",
-        "content",
-        "error",
-        "attempt_count",
-        "dead_lettered_at",
-        "profile_id",
-    }
-    return required <= columns