superlocalmemory 3.8.3 → 3.8.6

package/src/superlocalmemory/storage/migration_runner.py

@@ -122,6 +122,12 @@ from superlocalmemory.storage.migrations import (
 from superlocalmemory.storage.migrations import (
     M030_entity_explorer_indexes as _M030,
 )
+from superlocalmemory.storage.migrations import (
+    M031_dead_letter_operations as _M031,
+)
+from superlocalmemory.storage.migrations import (
+    M032_write_coordinator_admission as _M032,
+)
 
 # Map migration name → module (used for the optional ``verify(conn)`` hook
 # that lets the runner detect "already applied" state when an idempotent
@@ -156,6 +162,8 @@ _MODULES = {
     _M028.NAME: _M028,
     _M029.NAME: _M029,
     _M030.NAME: _M030,
+    _M031.NAME: _M031,
+    _M032.NAME: _M032,
 }
 
 logger = logging.getLogger(__name__)
@@ -169,6 +177,12 @@ _KNOWN_EQUIVALENT_DDL_HASHES: dict[str, frozenset[str]] = {
         # v3.4.22 model_version-default variant shipped through 3.6.x.
         "d28666fa1dfa66e6514efd288e6748363513da2255a4cee95d80f233e6728ae7",
     }),
+    _M032.NAME: frozenset({
+        # Provisional 3.8.6 development ledger: global idempotency_key and
+        # operation_id uniqueness. Its standalone table is safely rebuilt by
+        # M032.repair() into the profile-scoped receipt contract.
+        "e45df41becba3d0c3342eca5ec3bd83aa899eef76943c819d2da73b4ca1625a7",
+    }),
 }
 
 
@@ -216,6 +230,12 @@ MIGRATIONS: list[Migration] = [
     Migration(name=_M024.NAME, db_target="memory", ddl=_M024.DDL),
     Migration(name=_M019.NAME, db_target="memory", ddl=_M019.DDL,
               dependencies=(_M018.NAME,)),
+    # M031 creates dead_letter_operations — standalone table, no FK to engine-
+    # bootstrapped tables, so it can run during apply_all (before engine init).
+    Migration(name=_M031.NAME, db_target="memory", ddl=_M031.DDL),
+    # M032 is standalone and must precede daemon readiness: typed writes use
+    # this append-only receipt ledger for durable idempotency.
+    Migration(name=_M032.NAME, db_target="memory", ddl=_M032.DDL),
     # M006 + M011 are deliberately NOT here — see DEFERRED_MIGRATIONS below.
 ]
 
@@ -415,6 +435,30 @@ def _apply_single(
                             )
                     except sqlite3.Error:  # pragma: no cover
                         pass
+                    if dry_run:
+                        return (
+                            "skipped",
+                            "dry-run: would repair allowlisted historical schema",
+                        )
+                    repair_fn = getattr(mod, "repair", None) if mod is not None else None
+                    if callable(repair_fn):
+                        try:
+                            repair_fn(conn)
+                            if not bool(verify_fn(conn)):
+                                return (
+                                    "failed",
+                                    f"safe repair did not restore {migration.name}",
+                                )
+                            _upsert_log(conn, migration.name, ddl_hash, "complete")
+                            return (
+                                "applied",
+                                "allowlisted historical schema repaired safely",
+                            )
+                        except sqlite3.Error as exc:
+                            return (
+                                "failed",
+                                f"safe repair failed for {migration.name}: {exc}",
+                            )
                 detail = (
                     f"DDL drift detected for {migration.name}: "
                     f"logged={logged_hash[:8]}... current={ddl_hash[:8]}..."

package/src/superlocalmemory/storage/migrations/M028_fact_entity_associations.py

@@ -101,17 +101,30 @@ def _now() -> str:
     return datetime.now(UTC).isoformat()
 
 
-def _connect(db_path: Path) -> sqlite3.Connection:
-    conn = sqlite3.connect(str(db_path), timeout=5)
+def _connect_read(db_path: Path) -> sqlite3.Connection:
+    """Read-only connection with system-default busy_timeout.
+
+    Concurrency fix (v3.8.4): busy_timeout raised from 5 s to match the
+    system default (10 s, env SLM_DB_BUSY_TIMEOUT_MS).  Timeout raised from
+    5 s to 10 s for the same reason.  Write connections now go through
+    memory_write() so they acquire get_write_lock() and never use this helper.
+    """
+    import os
+
+    try:
+        ms = max(0, int(os.environ.get("SLM_DB_BUSY_TIMEOUT_MS", "10000")))
+    except (TypeError, ValueError):
+        ms = 10000
+    conn = sqlite3.connect(str(db_path), timeout=ms / 1000.0)
     conn.row_factory = sqlite3.Row
-    conn.execute("PRAGMA busy_timeout=5000")
+    conn.execute(f"PRAGMA busy_timeout={ms}")
     conn.execute("PRAGMA foreign_keys=ON")
     return conn
 
 
 def get_repair_status(db_path: Path) -> dict[str, int | str]:
     """Read durable backfill progress without inferring it from schema."""
-    conn = _connect(Path(db_path))
+    conn = _connect_read(Path(db_path))
     try:
         row = conn.execute(
             "SELECT state,target_fact_rowid,last_fact_rowid,scanned,inserted,"
@@ -142,58 +155,70 @@ def _entity_ids(raw: object) -> tuple[str, ...]:
 
 
 def _repair_batch(conn: sqlite3.Connection, batch_size: int) -> dict[str, int | bool]:
-    conn.execute("BEGIN IMMEDIATE")
-    try:
-        status = conn.execute(
-            "SELECT last_fact_rowid,target_fact_rowid "
-            "FROM fact_entity_association_repair_state "
-            "WHERE repair_key='historical-backfill'"
-        ).fetchone()
-        cursor = int(status["last_fact_rowid"] or 0)
-        target = int(status["target_fact_rowid"])
-        rows = conn.execute(
-            "SELECT rowid,fact_id,profile_id,canonical_entities_json "
-            "FROM atomic_facts WHERE rowid>? AND rowid<=? "
-            "ORDER BY rowid LIMIT ?",
-            (cursor, target, batch_size),
-        ).fetchall()
-        if not rows:
-            conn.execute(
-                "UPDATE fact_entity_association_repair_state "
-                "SET state='complete',last_error='',updated_at=? "
-                "WHERE repair_key='historical-backfill'",
-                (_now(),),
-            )
-            conn.commit()
-            return {"scanned": 0, "inserted": 0, "complete": True}
-        inserted = 0
-        for row in rows:
-            for entity_id in _entity_ids(row["canonical_entities_json"]):
-                result = conn.execute(
-                    "INSERT OR IGNORE INTO fact_entity_associations "
-                    "(profile_id,fact_id,entity_id,first_operation_id,"
-                    "count_applied) "
-                    "SELECT ?,?,?,?,? FROM canonical_entities "
-                    "WHERE profile_id=? AND entity_id=?",
-                    (
-                        row["profile_id"], row["fact_id"], entity_id,
-                        "migration-backfill", 0,
-                        row["profile_id"], entity_id,
-                    ),
-                )
-                inserted += max(0, result.rowcount)
+    """Execute one backfill batch on *conn*.
+
+    Concurrency fix (v3.8.4): caller MUST hold the write lock via
+    memory_write() before calling this function.  The explicit
+    ``conn.execute("BEGIN IMMEDIATE")`` has been removed because:
+
+    * memory_write() already acquired get_write_lock() (Python-level
+      serialisation) — no other in-process writer can start.
+    * SQLite's implicit deferred transaction is promoted to a write
+      transaction on the first INSERT, which is equivalent to BEGIN
+      IMMEDIATE for in-process serialisation.
+    * The explicit BEGIN IMMEDIATE previously bypassed get_write_lock()
+      and would retry at the SQLite layer only (busy_timeout=5 s, now
+      10 s) without respecting the Python-level write lock order.
+
+    Transaction boundary (commit/rollback) is managed by memory_write().
+    """
+    status = conn.execute(
+        "SELECT last_fact_rowid,target_fact_rowid "
+        "FROM fact_entity_association_repair_state "
+        "WHERE repair_key='historical-backfill'"
+    ).fetchone()
+    if status is None:
+        return {"scanned": 0, "inserted": 0, "complete": True}
+    cursor = int(status["last_fact_rowid"] or 0)
+    target = int(status["target_fact_rowid"])
+    rows = conn.execute(
+        "SELECT rowid,fact_id,profile_id,canonical_entities_json "
+        "FROM atomic_facts WHERE rowid>? AND rowid<=? "
+        "ORDER BY rowid LIMIT ?",
+        (cursor, target, batch_size),
+    ).fetchall()
+    if not rows:
         conn.execute(
-            "UPDATE fact_entity_association_repair_state SET "
-            "state='running',last_fact_rowid=?,scanned=scanned+?,"
-            "inserted=inserted+?,last_error='',updated_at=? "
+            "UPDATE fact_entity_association_repair_state "
+            "SET state='complete',last_error='',updated_at=? "
             "WHERE repair_key='historical-backfill'",
-            (int(rows[-1]["rowid"]), len(rows), inserted, _now()),
+            (_now(),),
         )
-        conn.commit()
-        return {"scanned": len(rows), "inserted": inserted, "complete": False}
-    except Exception:
-        conn.rollback()
-        raise
+        return {"scanned": 0, "inserted": 0, "complete": True}
+    inserted = 0
+    for row in rows:
+        for entity_id in _entity_ids(row["canonical_entities_json"]):
+            result = conn.execute(
+                "INSERT OR IGNORE INTO fact_entity_associations "
+                "(profile_id,fact_id,entity_id,first_operation_id,"
+                "count_applied) "
+                "SELECT ?,?,?,?,? FROM canonical_entities "
+                "WHERE profile_id=? AND entity_id=?",
+                (
+                    row["profile_id"], row["fact_id"], entity_id,
+                    "migration-backfill", 0,
+                    row["profile_id"], entity_id,
+                ),
+            )
+            inserted += max(0, result.rowcount)
+    conn.execute(
+        "UPDATE fact_entity_association_repair_state SET "
+        "state='running',last_fact_rowid=?,scanned=scanned+?,"
+        "inserted=inserted+?,last_error='',updated_at=? "
+        "WHERE repair_key='historical-backfill'",
+        (int(rows[-1]["rowid"]), len(rows), inserted, _now()),
+    )
+    return {"scanned": len(rows), "inserted": inserted, "complete": False}
 
 
 def repair_fact_entity_associations(
@@ -202,34 +227,44 @@ def repair_fact_entity_associations(
     batch_size: int = 250,
     max_batches: int = 1,
 ) -> dict[str, int | bool]:
-    """Run bounded, restartable short-transaction backfill batches."""
+    """Run bounded, restartable short-transaction backfill batches.
+
+    Concurrency fix (v3.8.4): each batch is now wrapped in memory_write()
+    which acquires get_write_lock() (process-level write serialisation) and
+    sets the system-default busy_timeout (10 s) before opening the SQLite
+    connection.  Previously the loop used a single raw _connect() (timeout=5,
+    busy_timeout=5000) without get_write_lock(), which bypassed in-process
+    write serialisation and could cause SQLITE_BUSY after only 5 s.
+    """
+    from superlocalmemory.storage.memory_write import memory_write
+
     if batch_size < 1 or max_batches < 1:
         raise ValueError("batch_size and max_batches must be positive")
-    totals = {"scanned": 0, "inserted": 0, "complete": False}
-    conn = _connect(Path(db_path))
-    try:
-        for _ in range(max_batches):
-            result = _repair_batch(conn, batch_size)
-            totals["scanned"] += int(result["scanned"])
-            totals["inserted"] += int(result["inserted"])
-            totals["complete"] = bool(result["complete"])
-            if totals["complete"]:
-                break
-        return totals
-    except sqlite3.Error as exc:
+    totals: dict[str, int | bool] = {"scanned": 0, "inserted": 0, "complete": False}
+    for _ in range(max_batches):
         try:
-            conn.execute(
-                "UPDATE fact_entity_association_repair_state SET "
-                "state='retrying',last_error=?,updated_at=? "
-                "WHERE repair_key='historical-backfill'",
-                (type(exc).__name__, _now()),
-            )
-            conn.commit()
-        except sqlite3.Error:
-            pass
-        raise
-    finally:
-        conn.close()
+            with memory_write(Path(db_path)) as conn:
+                conn.execute("PRAGMA foreign_keys=ON")
+                result = _repair_batch(conn, batch_size)
+                totals["scanned"] += int(result["scanned"])
+                totals["inserted"] += int(result["inserted"])
+                totals["complete"] = bool(result["complete"])
+        except sqlite3.Error as exc:
+            # On error, record the retrying state in a separate short write.
+            try:
+                with memory_write(Path(db_path)) as econn:
+                    econn.execute(
+                        "UPDATE fact_entity_association_repair_state SET "
+                        "state='retrying',last_error=?,updated_at=? "
+                        "WHERE repair_key='historical-backfill'",
+                        (type(exc).__name__, _now()),
+                    )
+            except sqlite3.Error:
+                pass
+            raise
+        if totals["complete"]:
+            break
+    return totals
 
 
 def verify(conn: sqlite3.Connection) -> bool:

package/src/superlocalmemory/storage/migrations/M031_dead_letter_operations.py

@@ -0,0 +1,80 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under AGPL-3.0-or-later - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""M031 — dead-letter queue for exhausted ingestion operations (Fix E, issue #77).
+
+Additive migration: creates dead_letter_operations if not present.
+Existing rows in ingestion_operations are unaffected.
+
+When an M018 ingestion operation exhausts _MAX_AUTOMATIC_MATERIALIZATION_ATTEMPTS
+(10) it previously remained silently in FAILED state — invisible to operators and
+unreachable by the materialiser.  This table gives operators a persistent,
+inspectable record of every poisoned operation: original content, error, attempt
+count, timestamps, and profile scope.
+
+Schema design:
+  - original_op_id references ingestion_operations.operation_id (soft ref — no FK
+    so that dead-lettered rows survive if the source row is later cleaned up).
+  - profile_id allows per-profile DLQ dashboards.
+  - dead_lettered_at defaults to the current epoch for point-in-time auditing.
+  - No TTL/expiry here — retention policy belongs to a future maintenance sweep.
+"""
+
+from __future__ import annotations
+
+import sqlite3
+
+NAME = "M031_dead_letter_operations"
+DB_TARGET = "memory"
+
+DDL = """
+CREATE TABLE IF NOT EXISTS dead_letter_operations (
+    id                  INTEGER PRIMARY KEY AUTOINCREMENT,
+    original_op_id      TEXT NOT NULL,
+    operation_type      TEXT NOT NULL DEFAULT 'M018',
+    content             TEXT,
+    metadata_json       TEXT,
+    error               TEXT,
+    attempt_count       INTEGER,
+    first_attempt_at    REAL,
+    dead_lettered_at    REAL NOT NULL DEFAULT (unixepoch('now')),
+    profile_id          TEXT
+);
+CREATE INDEX IF NOT EXISTS idx_dlq_profile
+    ON dead_letter_operations (profile_id);
+CREATE INDEX IF NOT EXISTS idx_dlq_op_id
+    ON dead_letter_operations (original_op_id);
+"""
+
+
+def apply(conn: sqlite3.Connection) -> None:
+    """Create the dead_letter_operations table and indexes idempotently."""
+    conn.executescript(DDL)
+
+
+def verify(conn: sqlite3.Connection) -> bool:
+    """Return True only when the complete M031 contract is present."""
+    table = conn.execute(
+        "SELECT 1 FROM sqlite_master WHERE type='table' "
+        "AND name='dead_letter_operations'"
+    ).fetchone()
+    if table is None:
+        return False
+    columns = {
+        row[1]
+        for row in conn.execute(
+            "PRAGMA table_info(dead_letter_operations)"
+        ).fetchall()
+    }
+    required = {
+        "id",
+        "original_op_id",
+        "operation_type",
+        "content",
+        "error",
+        "attempt_count",
+        "dead_lettered_at",
+        "profile_id",
+    }
+    return required <= columns

package/src/superlocalmemory/storage/migrations/M032_write_coordinator_admission.py

@@ -0,0 +1,188 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under AGPL-3.0-or-later - see LICENSE file
+
+"""M032 — durable, append-only receipts for canonical write commands.
+
+The receipt ledger is profile-isolated at the client idempotency boundary.
+``command_id`` and ``journal_id`` remain globally durable identifiers; a
+client-supplied idempotency key is unique only with its target profile.  An
+operation id is a projection label (for example, ``update:<fact_id>``), not a
+durable command identifier, so it is deliberately indexed but not unique.
+"""
+
+from __future__ import annotations
+
+import sqlite3
+
+NAME = "M032_write_coordinator_admission"
+DB_TARGET = "memory"
+
+DDL = """
+CREATE TABLE IF NOT EXISTS write_commits (
+    commit_sequence INTEGER PRIMARY KEY AUTOINCREMENT,
+    command_id      TEXT NOT NULL UNIQUE,
+    journal_id      TEXT NOT NULL UNIQUE,
+    command_kind    TEXT NOT NULL,
+    request_hash    TEXT NOT NULL,
+    profile_id      TEXT NOT NULL,
+    idempotency_key TEXT NOT NULL,
+    operation_id    TEXT NOT NULL,
+    receipt_json    TEXT NOT NULL,
+    committed_at    REAL NOT NULL,
+    UNIQUE(profile_id, idempotency_key)
+);
+CREATE INDEX IF NOT EXISTS idx_write_commits_committed_at
+    ON write_commits (committed_at);
+CREATE INDEX IF NOT EXISTS idx_write_commits_operation_id
+    ON write_commits (operation_id);
+CREATE TRIGGER IF NOT EXISTS trg_write_commits_immutable_update
+BEFORE UPDATE ON write_commits
+BEGIN
+    SELECT RAISE(ABORT, 'write_commits receipts are immutable');
+END;
+CREATE TRIGGER IF NOT EXISTS trg_write_commits_immutable_delete
+BEFORE DELETE ON write_commits
+BEGIN
+    SELECT RAISE(ABORT, 'write_commits receipts are immutable');
+END;
+"""
+
+_CREATE_TABLE = DDL.split(";", 1)[0]
+_CREATE_COMMITTED_AT_INDEX = (
+    "CREATE INDEX IF NOT EXISTS idx_write_commits_committed_at "
+    "ON write_commits (committed_at)"
+)
+_CREATE_OPERATION_ID_INDEX = (
+    "CREATE INDEX IF NOT EXISTS idx_write_commits_operation_id "
+    "ON write_commits (operation_id)"
+)
+_CREATE_UPDATE_TRIGGER = """
+CREATE TRIGGER IF NOT EXISTS trg_write_commits_immutable_update
+BEFORE UPDATE ON write_commits
+BEGIN
+    SELECT RAISE(ABORT, 'write_commits receipts are immutable');
+END
+"""
+_CREATE_DELETE_TRIGGER = """
+CREATE TRIGGER IF NOT EXISTS trg_write_commits_immutable_delete
+BEFORE DELETE ON write_commits
+BEGIN
+    SELECT RAISE(ABORT, 'write_commits receipts are immutable');
+END
+"""
+
+
+def apply(conn: sqlite3.Connection) -> None:
+    """Create or safely upgrade the profile-scoped append-only receipt ledger."""
+    if verify(conn):
+        return
+    if not _table_exists(conn):
+        _create_current_schema(conn)
+        return
+    _rebuild_legacy_schema(conn)
+
+
+def repair(conn: sqlite3.Connection) -> None:
+    """Repair a completed provisional M032 in developer/test databases."""
+    apply(conn)
+
+
+def verify(conn: sqlite3.Connection) -> bool:
+    """Return true only when the full profile-safe ledger contract exists."""
+    if not _table_exists(conn):
+        return False
+    columns = {row[1] for row in conn.execute("PRAGMA table_info(write_commits)").fetchall()}
+    required = {
+        "commit_sequence",
+        "command_id",
+        "journal_id",
+        "command_kind",
+        "request_hash",
+        "profile_id",
+        "idempotency_key",
+        "operation_id",
+        "receipt_json",
+        "committed_at",
+    }
+    if not required <= columns:
+        return False
+    if not _has_unique_index(conn, ("command_id",)):
+        return False
+    if not _has_unique_index(conn, ("journal_id",)):
+        return False
+    if not _has_unique_index(conn, ("profile_id", "idempotency_key")):
+        return False
+    if _has_unique_index(conn, ("idempotency_key",)):
+        return False
+    if _has_unique_index(conn, ("operation_id",)):
+        return False
+    object_names = {
+        row[0]
+        for row in conn.execute(
+            "SELECT name FROM sqlite_master WHERE name IN (?, ?, ?, ?)",
+            (
+                "idx_write_commits_committed_at",
+                "idx_write_commits_operation_id",
+                "trg_write_commits_immutable_update",
+                "trg_write_commits_immutable_delete",
+            ),
+        ).fetchall()
+    }
+    return object_names == {
+        "idx_write_commits_committed_at",
+        "idx_write_commits_operation_id",
+        "trg_write_commits_immutable_update",
+        "trg_write_commits_immutable_delete",
+    }
+
+
+def _table_exists(conn: sqlite3.Connection) -> bool:
+    return conn.execute(
+        "SELECT 1 FROM sqlite_master WHERE type='table' AND name='write_commits'"
+    ).fetchone() is not None
+
+
+def _has_unique_index(conn: sqlite3.Connection, columns: tuple[str, ...]) -> bool:
+    for index in conn.execute("PRAGMA index_list(write_commits)").fetchall():
+        if not index[2]:
+            continue
+        index_columns = tuple(
+            row[2] for row in conn.execute(f"PRAGMA index_info({index[1]})").fetchall()
+        )
+        if index_columns == columns:
+            return True
+    return False
+
+
+def _create_current_schema(conn: sqlite3.Connection) -> None:
+    conn.execute(_CREATE_TABLE)
+    conn.execute(_CREATE_COMMITTED_AT_INDEX)
+    conn.execute(_CREATE_OPERATION_ID_INDEX)
+    conn.execute(_CREATE_UPDATE_TRIGGER)
+    conn.execute(_CREATE_DELETE_TRIGGER)
+
+
+def _rebuild_legacy_schema(conn: sqlite3.Connection) -> None:
+    """Rebuild only M032's own standalone table under one savepoint."""
+    conn.execute("SAVEPOINT m032_profile_scoped_idempotency")
+    try:
+        conn.execute("DROP TRIGGER IF EXISTS trg_write_commits_immutable_update")
+        conn.execute("DROP TRIGGER IF EXISTS trg_write_commits_immutable_delete")
+        conn.execute("ALTER TABLE write_commits RENAME TO write_commits_legacy")
+        conn.execute("DROP INDEX IF EXISTS idx_write_commits_committed_at")
+        conn.execute("DROP INDEX IF EXISTS idx_write_commits_operation_id")
+        _create_current_schema(conn)
+        conn.execute(
+            "INSERT INTO write_commits("
+            "commit_sequence, command_id, journal_id, command_kind, request_hash, "
+            "profile_id, idempotency_key, operation_id, receipt_json, committed_at"
+            ") SELECT commit_sequence, command_id, journal_id, command_kind, request_hash, "
+            "profile_id, idempotency_key, operation_id, receipt_json, committed_at "
+            "FROM write_commits_legacy"
+        )
+        conn.execute("DROP TABLE write_commits_legacy")
+    except BaseException:
+        conn.execute("ROLLBACK TO m032_profile_scoped_idempotency")
+        conn.execute("RELEASE m032_profile_scoped_idempotency")
+        raise
+    conn.execute("RELEASE m032_profile_scoped_idempotency")