superlocalmemory 3.8.2 → 3.8.5

package/src/superlocalmemory/storage/migrations/M028_fact_entity_associations.py

@@ -101,17 +101,30 @@ def _now() -> str:
     return datetime.now(UTC).isoformat()
 
 
-def _connect(db_path: Path) -> sqlite3.Connection:
-    conn = sqlite3.connect(str(db_path), timeout=5)
+def _connect_read(db_path: Path) -> sqlite3.Connection:
+    """Read-only connection with system-default busy_timeout.
+
+    Concurrency fix (v3.8.4): busy_timeout raised from 5 s to match the
+    system default (10 s, env SLM_DB_BUSY_TIMEOUT_MS).  Timeout raised from
+    5 s to 10 s for the same reason.  Write connections now go through
+    memory_write() so they acquire get_write_lock() and never use this helper.
+    """
+    import os
+
+    try:
+        ms = max(0, int(os.environ.get("SLM_DB_BUSY_TIMEOUT_MS", "10000")))
+    except (TypeError, ValueError):
+        ms = 10000
+    conn = sqlite3.connect(str(db_path), timeout=ms / 1000.0)
     conn.row_factory = sqlite3.Row
-    conn.execute("PRAGMA busy_timeout=5000")
+    conn.execute(f"PRAGMA busy_timeout={ms}")
     conn.execute("PRAGMA foreign_keys=ON")
     return conn
 
 
 def get_repair_status(db_path: Path) -> dict[str, int | str]:
     """Read durable backfill progress without inferring it from schema."""
-    conn = _connect(Path(db_path))
+    conn = _connect_read(Path(db_path))
     try:
         row = conn.execute(
             "SELECT state,target_fact_rowid,last_fact_rowid,scanned,inserted,"
@@ -142,58 +155,70 @@ def _entity_ids(raw: object) -> tuple[str, ...]:
 
 
 def _repair_batch(conn: sqlite3.Connection, batch_size: int) -> dict[str, int | bool]:
-    conn.execute("BEGIN IMMEDIATE")
-    try:
-        status = conn.execute(
-            "SELECT last_fact_rowid,target_fact_rowid "
-            "FROM fact_entity_association_repair_state "
-            "WHERE repair_key='historical-backfill'"
-        ).fetchone()
-        cursor = int(status["last_fact_rowid"] or 0)
-        target = int(status["target_fact_rowid"])
-        rows = conn.execute(
-            "SELECT rowid,fact_id,profile_id,canonical_entities_json "
-            "FROM atomic_facts WHERE rowid>? AND rowid<=? "
-            "ORDER BY rowid LIMIT ?",
-            (cursor, target, batch_size),
-        ).fetchall()
-        if not rows:
-            conn.execute(
-                "UPDATE fact_entity_association_repair_state "
-                "SET state='complete',last_error='',updated_at=? "
-                "WHERE repair_key='historical-backfill'",
-                (_now(),),
-            )
-            conn.commit()
-            return {"scanned": 0, "inserted": 0, "complete": True}
-        inserted = 0
-        for row in rows:
-            for entity_id in _entity_ids(row["canonical_entities_json"]):
-                result = conn.execute(
-                    "INSERT OR IGNORE INTO fact_entity_associations "
-                    "(profile_id,fact_id,entity_id,first_operation_id,"
-                    "count_applied) "
-                    "SELECT ?,?,?,?,? FROM canonical_entities "
-                    "WHERE profile_id=? AND entity_id=?",
-                    (
-                        row["profile_id"], row["fact_id"], entity_id,
-                        "migration-backfill", 0,
-                        row["profile_id"], entity_id,
-                    ),
-                )
-                inserted += max(0, result.rowcount)
+    """Execute one backfill batch on *conn*.
+
+    Concurrency fix (v3.8.4): caller MUST hold the write lock via
+    memory_write() before calling this function.  The explicit
+    ``conn.execute("BEGIN IMMEDIATE")`` has been removed because:
+
+    * memory_write() already acquired get_write_lock() (Python-level
+      serialisation) — no other in-process writer can start.
+    * SQLite's implicit deferred transaction is promoted to a write
+      transaction on the first INSERT, which is equivalent to BEGIN
+      IMMEDIATE for in-process serialisation.
+    * The explicit BEGIN IMMEDIATE previously bypassed get_write_lock()
+      and would retry at the SQLite layer only (busy_timeout=5 s, now
+      10 s) without respecting the Python-level write lock order.
+
+    Transaction boundary (commit/rollback) is managed by memory_write().
+    """
+    status = conn.execute(
+        "SELECT last_fact_rowid,target_fact_rowid "
+        "FROM fact_entity_association_repair_state "
+        "WHERE repair_key='historical-backfill'"
+    ).fetchone()
+    if status is None:
+        return {"scanned": 0, "inserted": 0, "complete": True}
+    cursor = int(status["last_fact_rowid"] or 0)
+    target = int(status["target_fact_rowid"])
+    rows = conn.execute(
+        "SELECT rowid,fact_id,profile_id,canonical_entities_json "
+        "FROM atomic_facts WHERE rowid>? AND rowid<=? "
+        "ORDER BY rowid LIMIT ?",
+        (cursor, target, batch_size),
+    ).fetchall()
+    if not rows:
         conn.execute(
-            "UPDATE fact_entity_association_repair_state SET "
-            "state='running',last_fact_rowid=?,scanned=scanned+?,"
-            "inserted=inserted+?,last_error='',updated_at=? "
+            "UPDATE fact_entity_association_repair_state "
+            "SET state='complete',last_error='',updated_at=? "
             "WHERE repair_key='historical-backfill'",
-            (int(rows[-1]["rowid"]), len(rows), inserted, _now()),
+            (_now(),),
         )
-        conn.commit()
-        return {"scanned": len(rows), "inserted": inserted, "complete": False}
-    except Exception:
-        conn.rollback()
-        raise
+        return {"scanned": 0, "inserted": 0, "complete": True}
+    inserted = 0
+    for row in rows:
+        for entity_id in _entity_ids(row["canonical_entities_json"]):
+            result = conn.execute(
+                "INSERT OR IGNORE INTO fact_entity_associations "
+                "(profile_id,fact_id,entity_id,first_operation_id,"
+                "count_applied) "
+                "SELECT ?,?,?,?,? FROM canonical_entities "
+                "WHERE profile_id=? AND entity_id=?",
+                (
+                    row["profile_id"], row["fact_id"], entity_id,
+                    "migration-backfill", 0,
+                    row["profile_id"], entity_id,
+                ),
+            )
+            inserted += max(0, result.rowcount)
+    conn.execute(
+        "UPDATE fact_entity_association_repair_state SET "
+        "state='running',last_fact_rowid=?,scanned=scanned+?,"
+        "inserted=inserted+?,last_error='',updated_at=? "
+        "WHERE repair_key='historical-backfill'",
+        (int(rows[-1]["rowid"]), len(rows), inserted, _now()),
+    )
+    return {"scanned": len(rows), "inserted": inserted, "complete": False}
 
 
 def repair_fact_entity_associations(
@@ -202,34 +227,44 @@ def repair_fact_entity_associations(
     batch_size: int = 250,
     max_batches: int = 1,
 ) -> dict[str, int | bool]:
-    """Run bounded, restartable short-transaction backfill batches."""
+    """Run bounded, restartable short-transaction backfill batches.
+
+    Concurrency fix (v3.8.4): each batch is now wrapped in memory_write()
+    which acquires get_write_lock() (process-level write serialisation) and
+    sets the system-default busy_timeout (10 s) before opening the SQLite
+    connection.  Previously the loop used a single raw _connect() (timeout=5,
+    busy_timeout=5000) without get_write_lock(), which bypassed in-process
+    write serialisation and could cause SQLITE_BUSY after only 5 s.
+    """
+    from superlocalmemory.storage.memory_write import memory_write
+
     if batch_size < 1 or max_batches < 1:
         raise ValueError("batch_size and max_batches must be positive")
-    totals = {"scanned": 0, "inserted": 0, "complete": False}
-    conn = _connect(Path(db_path))
-    try:
-        for _ in range(max_batches):
-            result = _repair_batch(conn, batch_size)
-            totals["scanned"] += int(result["scanned"])
-            totals["inserted"] += int(result["inserted"])
-            totals["complete"] = bool(result["complete"])
-            if totals["complete"]:
-                break
-        return totals
-    except sqlite3.Error as exc:
+    totals: dict[str, int | bool] = {"scanned": 0, "inserted": 0, "complete": False}
+    for _ in range(max_batches):
         try:
-            conn.execute(
-                "UPDATE fact_entity_association_repair_state SET "
-                "state='retrying',last_error=?,updated_at=? "
-                "WHERE repair_key='historical-backfill'",
-                (type(exc).__name__, _now()),
-            )
-            conn.commit()
-        except sqlite3.Error:
-            pass
-        raise
-    finally:
-        conn.close()
+            with memory_write(Path(db_path)) as conn:
+                conn.execute("PRAGMA foreign_keys=ON")
+                result = _repair_batch(conn, batch_size)
+                totals["scanned"] += int(result["scanned"])
+                totals["inserted"] += int(result["inserted"])
+                totals["complete"] = bool(result["complete"])
+        except sqlite3.Error as exc:
+            # On error, record the retrying state in a separate short write.
+            try:
+                with memory_write(Path(db_path)) as econn:
+                    econn.execute(
+                        "UPDATE fact_entity_association_repair_state SET "
+                        "state='retrying',last_error=?,updated_at=? "
+                        "WHERE repair_key='historical-backfill'",
+                        (type(exc).__name__, _now()),
+                    )
+            except sqlite3.Error:
+                pass
+            raise
+        if totals["complete"]:
+            break
+    return totals
 
 
 def verify(conn: sqlite3.Connection) -> bool:

package/src/superlocalmemory/storage/migrations/M031_dead_letter_operations.py

@@ -0,0 +1,80 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under AGPL-3.0-or-later - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""M031 — dead-letter queue for exhausted ingestion operations (Fix E, issue #77).
+
+Additive migration: creates dead_letter_operations if not present.
+Existing rows in ingestion_operations are unaffected.
+
+When an M018 ingestion operation exhausts _MAX_AUTOMATIC_MATERIALIZATION_ATTEMPTS
+(10) it previously remained silently in FAILED state — invisible to operators and
+unreachable by the materialiser.  This table gives operators a persistent,
+inspectable record of every poisoned operation: original content, error, attempt
+count, timestamps, and profile scope.
+
+Schema design:
+  - original_op_id references ingestion_operations.operation_id (soft ref — no FK
+    so that dead-lettered rows survive if the source row is later cleaned up).
+  - profile_id allows per-profile DLQ dashboards.
+  - dead_lettered_at defaults to the current epoch for point-in-time auditing.
+  - No TTL/expiry here — retention policy belongs to a future maintenance sweep.
+"""
+
+from __future__ import annotations
+
+import sqlite3
+
+NAME = "M031_dead_letter_operations"
+DB_TARGET = "memory"
+
+DDL = """
+CREATE TABLE IF NOT EXISTS dead_letter_operations (
+    id                  INTEGER PRIMARY KEY AUTOINCREMENT,
+    original_op_id      TEXT NOT NULL,
+    operation_type      TEXT NOT NULL DEFAULT 'M018',
+    content             TEXT,
+    metadata_json       TEXT,
+    error               TEXT,
+    attempt_count       INTEGER,
+    first_attempt_at    REAL,
+    dead_lettered_at    REAL NOT NULL DEFAULT (unixepoch('now')),
+    profile_id          TEXT
+);
+CREATE INDEX IF NOT EXISTS idx_dlq_profile
+    ON dead_letter_operations (profile_id);
+CREATE INDEX IF NOT EXISTS idx_dlq_op_id
+    ON dead_letter_operations (original_op_id);
+"""
+
+
+def apply(conn: sqlite3.Connection) -> None:
+    """Create the dead_letter_operations table and indexes idempotently."""
+    conn.executescript(DDL)
+
+
+def verify(conn: sqlite3.Connection) -> bool:
+    """Return True only when the complete M031 contract is present."""
+    table = conn.execute(
+        "SELECT 1 FROM sqlite_master WHERE type='table' "
+        "AND name='dead_letter_operations'"
+    ).fetchone()
+    if table is None:
+        return False
+    columns = {
+        row[1]
+        for row in conn.execute(
+            "PRAGMA table_info(dead_letter_operations)"
+        ).fetchall()
+    }
+    required = {
+        "id",
+        "original_op_id",
+        "operation_type",
+        "content",
+        "error",
+        "attempt_count",
+        "dead_lettered_at",
+        "profile_id",
+    }
+    return required <= columns

package/src/superlocalmemory/storage/write_lock.py

@@ -0,0 +1,88 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under AGPL-3.0-or-later - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com
+
+"""Process-level per-path write-lock registry for memory.db.
+
+ALL code that writes to a given SQLite file MUST acquire the lock returned
+by ``get_write_lock(db_path)`` BEFORE opening a sqlite3 connection for
+writing.  This serialises writes at the Python layer so that only ONE
+sqlite3 connection holds the SQLite WAL write lock at any instant, which
+eliminates all SQLITE_BUSY retries within the process.
+
+Invariant
+---------
+Every memory.db write path in the process acquires the same RLock before
+touching sqlite3.  Because SQLite WAL mode allows one writer at a time, a
+Python-level RLock gives us the same serialisation guarantee while
+completely avoiding the SQLITE_BUSY / busy_timeout retry loop.
+
+Lock ordering rule (must never be violated)
+-------------------------------------------
+    get_write_lock(memory.db)  ←  OUTERMOST
+        ↳  VectorStore._lock   (inner — VectorStore in-memory state)
+            ↳  sqlite3 write transaction (BEGIN … COMMIT)
+
+Re-entrancy
+-----------
+``threading.RLock`` is used so that the same thread can re-acquire the
+lock without deadlocking.  This is required by the self-heal backfill
+pattern::
+
+    with db._lock:          # db._lock IS get_write_lock(memory.db)
+        vs.upsert(...)      # also calls get_write_lock → re-entrant OK
+
+Usage
+-----
+    from superlocalmemory.storage.write_lock import get_write_lock
+
+    with get_write_lock(db_path):
+        conn = sqlite3.connect(str(db_path))
+        conn.execute("INSERT INTO ...")
+        conn.commit()
+        conn.close()
+"""
+from __future__ import annotations
+
+import threading
+from pathlib import Path
+
+# Registry: resolved absolute path string → RLock.
+# Protected by its own Lock so the registry itself is thread-safe.
+_registry: dict[str, threading.RLock] = {}
+_registry_lock = threading.Lock()
+
+
+def get_write_lock(db_path: str | Path) -> threading.RLock:
+    """Return the process-level write-serialisation RLock for *db_path*.
+
+    Resolves the path to its canonical absolute form before hashing so
+    that relative paths, ``.`` components, and symlinks all map to the
+    same lock as their resolved target.  Creates a new ``RLock`` on first
+    call for each unique resolved path.  Thread-safe.
+
+    Parameters
+    ----------
+    db_path:
+        Path to the SQLite database file.  The file need not exist yet.
+
+    Returns
+    -------
+    threading.RLock
+        The shared write-serialisation lock for *db_path*.  All writers
+        of this file in the current process share the same object.
+    """
+    p = Path(db_path)
+    try:
+        key = str(p.resolve())
+    except OSError:
+        # File doesn't exist yet; use absolute path as best-effort key.
+        key = str(p.absolute())
+
+    with _registry_lock:
+        if key not in _registry:
+            _registry[key] = threading.RLock()
+        return _registry[key]
+
+
+__all__ = ["get_write_lock"]

package/src/superlocalmemory/ui/js/core.js

@@ -13,7 +13,12 @@
 // surfaces as a normal rejection and the UI can show a clear error.
 // ============================================================================
 
-window.SLM_FETCH_TIMEOUT_MS = 15000;
+// v3.8.3: 30s (> the server's 25s recall budget) so the browser waits for a
+// quality recall under heavy multi-agent load instead of aborting it early
+// ("signal is aborted without reason"). The server self-bounds recall and
+// falls back to keyword within its budget, so this only ever waits longer for
+// a genuinely stuck request. Per-call init.timeoutMs still overrides.
+window.SLM_FETCH_TIMEOUT_MS = 30000;
 window.SLM_INSTALL_TOKEN_KEY = 'slm_install_token';
 
 // B2 (3.7.9): the install token is kept in a private closure, never in