superlocalmemory 3.8.3 → 3.8.6

package/src/superlocalmemory/learning/memory_merge.py

@@ -29,6 +29,8 @@ import uuid
 from datetime import datetime, timezone
 from pathlib import Path
 
+from superlocalmemory.storage.write_lock import get_write_lock
+
 logger = logging.getLogger(__name__)
 
 
@@ -55,64 +57,67 @@ def apply_merges(
     if not candidates:
         return 0
 
-    conn = sqlite3.connect(str(memory_db_path), timeout=10.0)
-    conn.execute("PRAGMA busy_timeout=2000")
+    # Acquire the process-level write lock BEFORE opening the sqlite3
+    # connection.  This serialises BEGIN IMMEDIATE → commit with all other
+    # in-process writers (DatabaseManager, VectorStore, adapter sync,
+    # reward_archive), eliminating SQLITE_BUSY races at the WAL layer.
     applied = 0
-    # S-L02: track the candidate list in flight so a rollback diagnostic
-    # can blame the exact set of (canonical, merged) pairs instead of a
-    # blanket "rollback" message. Operators on the dashboard previously
-    # saw zero fidelity about which candidates were in the transaction
-    # at commit-time.
-    in_flight: list[tuple[str, str]] = []
-    try:
-        conn.execute("BEGIN IMMEDIATE")
-        for canonical_id, merged_id, cos, jac in candidates:
-            # Skip if already merged in a prior cycle.
-            row = conn.execute(
-                "SELECT archive_status FROM atomic_facts WHERE fact_id=?",
-                (merged_id,),
-            ).fetchone()
-            if row is None:
-                continue
-            if row[0] == "merged":
-                continue
-
-            conn.execute(
-                "INSERT INTO memory_merge_log "
-                "(merge_id, profile_id, canonical_fact_id, merged_fact_id, "
-                " cosine_sim, entity_jaccard, merged_at, reversible) "
-                "VALUES (?, ?, ?, ?, ?, ?, ?, 1)",
-                (
-                    str(uuid.uuid4()),
-                    profile_id,
-                    canonical_id,
-                    merged_id,
-                    float(cos),
-                    float(jac),
-                    _iso_now(),
-                ),
-            )
-            conn.execute(
-                "UPDATE atomic_facts "
-                "SET archive_status='merged', "
-                "    archive_reason='cosine_dup', "
-                "    merged_into=? "
-                "WHERE fact_id=?",
-                (canonical_id, merged_id),
+    with get_write_lock(memory_db_path):
+        conn = sqlite3.connect(str(memory_db_path), timeout=10.0)
+        conn.execute("PRAGMA busy_timeout=2000")
+        # S-L02: track the candidate list in flight so a rollback diagnostic
+        # can blame the exact set of (canonical, merged) pairs instead of a
+        # blanket "rollback" message.
+        in_flight: list[tuple[str, str]] = []
+        try:
+            conn.execute("BEGIN IMMEDIATE")
+            for canonical_id, merged_id, cos, jac in candidates:
+                # Skip if already merged in a prior cycle.
+                row = conn.execute(
+                    "SELECT archive_status FROM atomic_facts WHERE fact_id=?",
+                    (merged_id,),
+                ).fetchone()
+                if row is None:
+                    continue
+                if row[0] == "merged":
+                    continue
+
+                conn.execute(
+                    "INSERT INTO memory_merge_log "
+                    "(merge_id, profile_id, canonical_fact_id, merged_fact_id, "
+                    " cosine_sim, entity_jaccard, merged_at, reversible) "
+                    "VALUES (?, ?, ?, ?, ?, ?, ?, 1)",
+                    (
+                        str(uuid.uuid4()),
+                        profile_id,
+                        canonical_id,
+                        merged_id,
+                        float(cos),
+                        float(jac),
+                        _iso_now(),
+                    ),
+                )
+                conn.execute(
+                    "UPDATE atomic_facts "
+                    "SET archive_status='merged', "
+                    "    archive_reason='cosine_dup', "
+                    "    merged_into=? "
+                    "WHERE fact_id=?",
+                    (canonical_id, merged_id),
+                )
+                applied += 1
+                in_flight.append((canonical_id, merged_id))
+            conn.commit()
+        except sqlite3.Error as exc:
+            conn.rollback()
+            logger.warning(
+                "apply_merges rollback: profile=%s pre-rollback_applied=%d "
+                "in_flight=%s error=%s",
+                profile_id, applied, in_flight, exc,
             )
-            applied += 1
-            in_flight.append((canonical_id, merged_id))
-        conn.commit()
-    except sqlite3.Error as exc:
-        conn.rollback()
-        logger.warning(
-            "apply_merges rollback: profile=%s pre-rollback_applied=%d "
-            "in_flight=%s error=%s",
-            profile_id, applied, in_flight, exc,
-        )
-        applied = 0
-    finally:
-        conn.close()
+            applied = 0
+        finally:
+            conn.close()
     return applied
 
 
@@ -122,39 +127,49 @@ def unmerge(memory_db_path: str | Path, merge_id: str) -> bool:
     Flips the merged fact's archive_status back to 'live', clears
     merged_into, and marks the log row ``reversible=0``.
     """
-    conn = sqlite3.connect(str(memory_db_path), timeout=10.0)
-    conn.execute("PRAGMA busy_timeout=2000")
+    # READ phase — check reversibility without holding the write lock.
+    conn_r = sqlite3.connect(str(memory_db_path), timeout=10.0)
     try:
-        row = conn.execute(
+        row = conn_r.execute(
             "SELECT merged_fact_id, reversible FROM memory_merge_log "
             "WHERE merge_id=?",
             (merge_id,),
         ).fetchone()
-        if row is None:
-            return False
-        merged_fid, reversible = row
-        if not reversible:
-            return False
+    finally:
+        conn_r.close()
 
-        conn.execute("BEGIN IMMEDIATE")
-        conn.execute(
-            "UPDATE atomic_facts "
-            "SET archive_status='live', archive_reason=NULL, merged_into=NULL "
-            "WHERE fact_id=?",
-            (merged_fid,),
-        )
-        conn.execute(
-            "UPDATE memory_merge_log SET reversible=0 WHERE merge_id=?",
-            (merge_id,),
-        )
-        conn.commit()
-        return True
-    except sqlite3.Error as exc:
-        conn.rollback()
-        logger.warning("unmerge rollback: %s", exc)
+    if row is None:
         return False
-    finally:
-        conn.close()
+    merged_fid, reversible = row
+    if not reversible:
+        return False
+
+    # WRITE phase — acquire the process-level write lock before
+    # opening the write connection, serialising with all other in-process
+    # writers and eliminating SQLITE_BUSY at the WAL layer.
+    with get_write_lock(memory_db_path):
+        conn = sqlite3.connect(str(memory_db_path), timeout=10.0)
+        conn.execute("PRAGMA busy_timeout=2000")
+        try:
+            conn.execute("BEGIN IMMEDIATE")
+            conn.execute(
+                "UPDATE atomic_facts "
+                "SET archive_status='live', archive_reason=NULL, merged_into=NULL "
+                "WHERE fact_id=?",
+                (merged_fid,),
+            )
+            conn.execute(
+                "UPDATE memory_merge_log SET reversible=0 WHERE merge_id=?",
+                (merge_id,),
+            )
+            conn.commit()
+            return True
+        except sqlite3.Error as exc:
+            conn.rollback()
+            logger.warning("unmerge rollback: %s", exc)
+            return False
+        finally:
+            conn.close()
 
 
 __all__ = ("apply_merges", "unmerge")

package/src/superlocalmemory/learning/reward_archive.py

@@ -34,6 +34,7 @@ from pathlib import Path
 from superlocalmemory.learning.fact_outcome_joins import (
     has_recent_positive_reward,
 )
+from superlocalmemory.storage.write_lock import get_write_lock
 
 logger = logging.getLogger(__name__)
 
@@ -80,13 +81,16 @@ def run_reward_gated_archive(
         return []
 
     archived: list[str] = []
-    conn = sqlite3.connect(str(memory_db_path), timeout=10.0)
-    conn.row_factory = sqlite3.Row
-    conn.execute("PRAGMA busy_timeout=2000")
 
+    # READ phase — no write lock needed (WAL allows concurrent reads).
+    # H-12/H-P-02: compute the archivable set BEFORE acquiring the writer
+    # lock so that the lock is held only for the bulk INSERT/UPDATE pass.
+    conn_r = sqlite3.connect(str(memory_db_path), timeout=10.0)
+    conn_r.row_factory = sqlite3.Row
+    conn_r.execute("PRAGMA busy_timeout=2000")
     try:
         placeholders = ",".join("?" for _ in candidate_fact_ids)
-        rows = conn.execute(
+        rows = conn_r.execute(
             f"SELECT fact_id, content, canonical_entities_json, importance, "
             f"       confidence, embedding, created_at "
             f"FROM atomic_facts "
@@ -95,13 +99,6 @@ def run_reward_gated_archive(
             (profile_id, *candidate_fact_ids),
         ).fetchall()
 
-        # H-12/H-P-02: compute the archivable set BEFORE acquiring the
-        # writer lock. Previously ``BEGIN IMMEDIATE`` wrapped the full
-        # per-candidate reward-lookup loop — holding RESERVED for the
-        # entire scan starved concurrent ``record_recall`` writers out
-        # with SQLITE_BUSY after their 50 ms busy_timeout. Splitting the
-        # read phase keeps the writer lock held only for the bulk
-        # INSERT/UPDATE pass.
         to_archive: list[dict] = []
         for row in rows:
             fid = row["fact_id"]
@@ -111,7 +108,7 @@ def run_reward_gated_archive(
             # 2. Recent positive reward skip (criterion 2).
             #    H-06 fix — JSON1 equality join via helper.
             if has_recent_positive_reward(
-                conn, profile_id, fid,
+                conn_r, profile_id, fid,
                 min_reward=ARCHIVE_REWARD_THRESHOLD,
                 window_days=REWARD_WINDOW_DAYS,
             ):
@@ -125,86 +122,97 @@ def run_reward_gated_archive(
                 "embedding": row["embedding"],
                 "created_at": row["created_at"],
             })
+    finally:
+        conn_r.close()
 
-        if not to_archive:
-            return []
-
-        conn.execute("BEGIN IMMEDIATE")
-        # S9-SKEP-07: re-verify reward under RESERVED lock. Between the
-        # read-only reward scan above and this BEGIN IMMEDIATE another
-        # writer may have inserted a positive reward row ("user liked
-        # the memory we are about to archive"). With the writer lock
-        # held we now know no further inserts are landing; one last
-        # check per entry catches everything that raced in.
-        verified: list[dict] = []
-        for entry in to_archive:
-            if has_recent_positive_reward(
-                conn, profile_id, entry["fid"],
-                min_reward=ARCHIVE_REWARD_THRESHOLD,
-                window_days=REWARD_WINDOW_DAYS,
-            ):
-                continue
-            verified.append(entry)
-        to_archive = verified
-        if not to_archive:
-            conn.execute("COMMIT")
-            return []
-        for entry in to_archive:
-            fid = entry["fid"]
-            payload = {
-                "fact_id": fid,
-                "content": entry["content"],
-                "canonical_entities_json": entry["canonical_entities_json"],
-                "importance": entry["importance"],
-                "confidence": entry["confidence"],
-                "embedding": entry["embedding"],
-                "created_at": entry["created_at"],
-            }
-            # SEC-L3 — cap payload_json at 256 KB. Oversize blobs are
-            # replaced with a minimal stub + ``truncated`` reason so the
-            # archive row stays within the I4 disk budget while still
-            # pointing back to the original ``fact_id`` in atomic_facts.
-            payload_str = json.dumps(payload)
-            reason = "reward_gated_ebbinghaus"
-            if len(payload_str.encode("utf-8")) > PAYLOAD_JSON_MAX_BYTES:
-                payload_str = json.dumps({
+    if not to_archive:
+        return []
+
+    # WRITE phase — acquire the process-level write lock BEFORE opening the
+    # write connection.  This serialises BEGIN IMMEDIATE → commit with all
+    # other in-process writers (DatabaseManager, VectorStore, adapter sync,
+    # memory_merge, reward_boost), eliminating SQLITE_BUSY races at the WAL
+    # layer.
+    with get_write_lock(memory_db_path):
+        conn = sqlite3.connect(str(memory_db_path), timeout=10.0)
+        conn.row_factory = sqlite3.Row
+        conn.execute("PRAGMA busy_timeout=2000")
+        try:
+            conn.execute("BEGIN IMMEDIATE")
+            # S9-SKEP-07: re-verify reward under RESERVED lock. Between the
+            # read-only reward scan above and this BEGIN IMMEDIATE another
+            # writer may have inserted a positive reward row. With the writer
+            # lock held we know no further inserts are landing; one last
+            # check per entry catches everything that raced in.
+            verified: list[dict] = []
+            for entry in to_archive:
+                if has_recent_positive_reward(
+                    conn, profile_id, entry["fid"],
+                    min_reward=ARCHIVE_REWARD_THRESHOLD,
+                    window_days=REWARD_WINDOW_DAYS,
+                ):
+                    continue
+                verified.append(entry)
+            to_archive = verified
+            if not to_archive:
+                conn.execute("COMMIT")
+                return []
+            for entry in to_archive:
+                fid = entry["fid"]
+                payload = {
                     "fact_id": fid,
-                    "truncated": True,
-                    "original_bytes": len(payload_str.encode("utf-8")),
-                })
-                reason = "reward_gated_ebbinghaus_truncated"
-                logger.warning(
-                    "memory_archive payload >%d bytes for fact_id=%s; "
-                    "truncated to stub", PAYLOAD_JSON_MAX_BYTES, fid,
+                    "content": entry["content"],
+                    "canonical_entities_json": entry["canonical_entities_json"],
+                    "importance": entry["importance"],
+                    "confidence": entry["confidence"],
+                    "embedding": entry["embedding"],
+                    "created_at": entry["created_at"],
+                }
+                # SEC-L3 — cap payload_json at 256 KB. Oversize blobs are
+                # replaced with a minimal stub + ``truncated`` reason so the
+                # archive row stays within the I4 disk budget while still
+                # pointing back to the original ``fact_id`` in atomic_facts.
+                payload_str = json.dumps(payload)
+                reason = "reward_gated_ebbinghaus"
+                if len(payload_str.encode("utf-8")) > PAYLOAD_JSON_MAX_BYTES:
+                    payload_str = json.dumps({
+                        "fact_id": fid,
+                        "truncated": True,
+                        "original_bytes": len(payload_str.encode("utf-8")),
+                    })
+                    reason = "reward_gated_ebbinghaus_truncated"
+                    logger.warning(
+                        "memory_archive payload >%d bytes for fact_id=%s; "
+                        "truncated to stub", PAYLOAD_JSON_MAX_BYTES, fid,
+                    )
+                conn.execute(
+                    "INSERT INTO memory_archive "
+                    "(archive_id, fact_id, profile_id, payload_json, "
+                    " archived_at, reason) VALUES (?, ?, ?, ?, ?, ?)",
+                    (
+                        str(uuid.uuid4()),
+                        fid,
+                        profile_id,
+                        payload_str,
+                        _iso_now(),
+                        reason,
+                    ),
                 )
-            conn.execute(
-                "INSERT INTO memory_archive "
-                "(archive_id, fact_id, profile_id, payload_json, "
-                " archived_at, reason) VALUES (?, ?, ?, ?, ?, ?)",
-                (
-                    str(uuid.uuid4()),
-                    fid,
-                    profile_id,
-                    payload_str,
-                    _iso_now(),
-                    reason,
-                ),
-            )
-            conn.execute(
-                "UPDATE atomic_facts "
-                "SET archive_status='archived', "
-                "    archive_reason='reward_gated_ebbinghaus' "
-                "WHERE fact_id=? "
-                "  AND (archive_status IS NULL OR archive_status='live')",
-                (fid,),
-            )
-            archived.append(fid)
-
-        conn.commit()
-    except sqlite3.Error as exc:
-        conn.rollback()
-        logger.warning("run_reward_gated_archive rollback: %s", exc)
-    finally:
-        conn.close()
+                conn.execute(
+                    "UPDATE atomic_facts "
+                    "SET archive_status='archived', "
+                    "    archive_reason='reward_gated_ebbinghaus' "
+                    "WHERE fact_id=? "
+                    "  AND (archive_status IS NULL OR archive_status='live')",
+                    (fid,),
+                )
+                archived.append(fid)
+
+            conn.commit()
+        except sqlite3.Error as exc:
+            conn.rollback()
+            logger.warning("run_reward_gated_archive rollback: %s", exc)
+        finally:
+            conn.close()
 
     return archived

package/src/superlocalmemory/learning/reward_boost.py

@@ -23,6 +23,7 @@ from pathlib import Path
 from superlocalmemory.learning.fact_outcome_joins import (
     aggregate_reward_for_fact,
 )
+from superlocalmemory.storage.write_lock import get_write_lock
 
 
 # H-12/H-P-01: single-pass JSON1 aggregation across ALL facts for a profile.
@@ -123,45 +124,54 @@ def apply_strong_memory_boost(
 
     Returns number of rows boosted.
     """
-    conn = sqlite3.connect(str(memory_db_path), timeout=10.0)
-    conn.execute("PRAGMA busy_timeout=2000")
-    boosted = 0
+    # READ phase — no write lock needed (WAL allows concurrent reads).
+    conn_r = sqlite3.connect(str(memory_db_path), timeout=10.0)
     try:
-        rows = conn.execute(
+        rows = conn_r.execute(
             "SELECT fact_id FROM atomic_facts WHERE profile_id=? "
             "  AND (archive_status IS NULL OR archive_status='live')",
             (profile_id,),
         ).fetchall()
         if not rows:
+            conn_r.close()
             return 0
-
         # H-12/H-P-01: single JSON1 GROUP BY replaces the per-fact loop.
-        # Fallback to per-fact helper preserves legacy behaviour on
-        # SQLite without JSON1.
-        stats = _bulk_fact_reward_stats(conn, profile_id)
-        conn.execute("BEGIN IMMEDIATE")
-        for (fid,) in rows:
-            if stats:
-                count, mean = stats.get(fid, _MISS)
-            else:
-                count, mean = aggregate_reward_for_fact(conn, profile_id, fid)
-            if count < STRONG_BOOST_MIN_OUTCOMES:
-                continue
-            if mean <= STRONG_BOOST_MIN_MEAN:
-                continue
-            conn.execute(
-                "UPDATE atomic_facts "
-                "SET retrieval_prior = MIN(COALESCE(retrieval_prior, 0) + ?, ?) "
-                "WHERE fact_id=?",
-                (STRONG_BOOST_INCREMENT, STRONG_BOOST_CAP, fid),
-            )
-            boosted += 1
-        conn.commit()
-    except sqlite3.Error as exc:
-        conn.rollback()
-        logger.warning("apply_strong_memory_boost rollback: %s", exc)
+        stats = _bulk_fact_reward_stats(conn_r, profile_id)
     finally:
-        conn.close()
+        conn_r.close()
+
+    # WRITE phase — acquire the process-level write lock BEFORE opening the
+    # write connection.  This serialises BEGIN IMMEDIATE → commit with all
+    # other in-process writers (DatabaseManager, VectorStore, adapter sync),
+    # eliminating SQLITE_BUSY races at the SQLite WAL layer.
+    boosted = 0
+    with get_write_lock(memory_db_path):
+        conn = sqlite3.connect(str(memory_db_path), timeout=10.0)
+        conn.execute("PRAGMA busy_timeout=2000")
+        try:
+            conn.execute("BEGIN IMMEDIATE")
+            for (fid,) in rows:
+                if stats:
+                    count, mean = stats.get(fid, _MISS)
+                else:
+                    count, mean = aggregate_reward_for_fact(conn, profile_id, fid)
+                if count < STRONG_BOOST_MIN_OUTCOMES:
+                    continue
+                if mean <= STRONG_BOOST_MIN_MEAN:
+                    continue
+                conn.execute(
+                    "UPDATE atomic_facts "
+                    "SET retrieval_prior = MIN(COALESCE(retrieval_prior, 0) + ?, ?) "
+                    "WHERE fact_id=?",
+                    (STRONG_BOOST_INCREMENT, STRONG_BOOST_CAP, fid),
+                )
+                boosted += 1
+            conn.commit()
+        except sqlite3.Error as exc:
+            conn.rollback()
+            logger.warning("apply_strong_memory_boost rollback: %s", exc)
+        finally:
+            conn.close()
     return boosted
 
 

package/src/superlocalmemory/learning/source_quality.py

@@ -81,6 +81,7 @@ CREATE TABLE IF NOT EXISTS source_quality_repair_state (
 _MAX_FACTS_PER_OUTCOME = 100
 _MAX_SOURCES_PER_OUTCOME = 100
 _PROVENANCE_QUERY_CHUNK = 500
+_SCHEMA_INIT_LOCK = threading.RLock()
 
 
 class SourceQualityRepairUnavailable(RuntimeError):
@@ -114,45 +115,47 @@ class SourceQualityScorer:
     # ------------------------------------------------------------------
 
     def _ensure_schema(self) -> None:
-        conn = self._connect()
-        try:
-            # Separate scorer instances can be constructed concurrently during
-            # first startup (background history repair + outcome settlement).
-            # Serialize the read/ALTER sequence at SQLite's transaction
-            # boundary so two processes cannot both observe a legacy column as
-            # missing and race into ``duplicate column name``.
-            conn.execute("BEGIN IMMEDIATE")
-            conn.execute(_CREATE_TABLE)
-            conn.execute(_CREATE_UNIQUE)
-            conn.execute(_CREATE_OBSERVATIONS)
-            conn.execute(_CREATE_REPAIR_STATE)
-            repair_columns = {
-                str(row["name"])
-                for row in conn.execute(
-                    "PRAGMA table_info(source_quality_repair_state)"
-                ).fetchall()
-            }
-            if "last_settled_at" not in repair_columns:
-                conn.execute(
-                    "ALTER TABLE source_quality_repair_state "
-                    "ADD COLUMN last_settled_at TEXT NOT NULL DEFAULT ''"
-                )
-            if "last_outcome_id" not in repair_columns:
-                conn.execute(
-                    "ALTER TABLE source_quality_repair_state "
-                    "ADD COLUMN last_outcome_id TEXT NOT NULL DEFAULT ''"
-                )
-            conn.commit()
-        except Exception:
-            conn.rollback()
-            raise
-        finally:
-            conn.close()
+        # Serialize scorer instances in this process before asking SQLite for
+        # its cross-process BEGIN IMMEDIATE lease. This prevents two startup
+        # threads from racing the journal-mode/schema bootstrap while SQLite's
+        # busy timeout protects the equivalent multi-process boundary.
+        with _SCHEMA_INIT_LOCK:
+            conn = self._connect()
+            try:
+                conn.execute("BEGIN IMMEDIATE")
+                conn.execute(_CREATE_TABLE)
+                conn.execute(_CREATE_UNIQUE)
+                conn.execute(_CREATE_OBSERVATIONS)
+                conn.execute(_CREATE_REPAIR_STATE)
+                repair_columns = {
+                    str(row["name"])
+                    for row in conn.execute(
+                        "PRAGMA table_info(source_quality_repair_state)"
+                    ).fetchall()
+                }
+                if "last_settled_at" not in repair_columns:
+                    conn.execute(
+                        "ALTER TABLE source_quality_repair_state "
+                        "ADD COLUMN last_settled_at TEXT NOT NULL DEFAULT ''"
+                    )
+                if "last_outcome_id" not in repair_columns:
+                    conn.execute(
+                        "ALTER TABLE source_quality_repair_state "
+                        "ADD COLUMN last_outcome_id TEXT NOT NULL DEFAULT ''"
+                    )
+                conn.commit()
+            except Exception:
+                conn.rollback()
+                raise
+            finally:
+                conn.close()
 
     def _connect(self) -> sqlite3.Connection:
         conn = sqlite3.connect(str(self._db_path), timeout=10)
+        # Install the busy handler before journal negotiation. On a fresh
+        # database, PRAGMA journal_mode itself may contend with another scorer.
+        conn.execute("PRAGMA busy_timeout=10000")
         conn.execute("PRAGMA journal_mode=WAL")
-        conn.execute("PRAGMA busy_timeout=5000")
         conn.row_factory = sqlite3.Row
         return conn