superlocalmemory 3.8.2 → 3.8.5

package/src/superlocalmemory/learning/reward_archive.py

@@ -34,6 +34,7 @@ from pathlib import Path
 from superlocalmemory.learning.fact_outcome_joins import (
     has_recent_positive_reward,
 )
+from superlocalmemory.storage.write_lock import get_write_lock
 
 logger = logging.getLogger(__name__)
 
@@ -80,13 +81,16 @@ def run_reward_gated_archive(
         return []
 
     archived: list[str] = []
-    conn = sqlite3.connect(str(memory_db_path), timeout=10.0)
-    conn.row_factory = sqlite3.Row
-    conn.execute("PRAGMA busy_timeout=2000")
 
+    # READ phase — no write lock needed (WAL allows concurrent reads).
+    # H-12/H-P-02: compute the archivable set BEFORE acquiring the writer
+    # lock so that the lock is held only for the bulk INSERT/UPDATE pass.
+    conn_r = sqlite3.connect(str(memory_db_path), timeout=10.0)
+    conn_r.row_factory = sqlite3.Row
+    conn_r.execute("PRAGMA busy_timeout=2000")
     try:
         placeholders = ",".join("?" for _ in candidate_fact_ids)
-        rows = conn.execute(
+        rows = conn_r.execute(
             f"SELECT fact_id, content, canonical_entities_json, importance, "
             f"       confidence, embedding, created_at "
             f"FROM atomic_facts "
@@ -95,13 +99,6 @@ def run_reward_gated_archive(
             (profile_id, *candidate_fact_ids),
         ).fetchall()
 
-        # H-12/H-P-02: compute the archivable set BEFORE acquiring the
-        # writer lock. Previously ``BEGIN IMMEDIATE`` wrapped the full
-        # per-candidate reward-lookup loop — holding RESERVED for the
-        # entire scan starved concurrent ``record_recall`` writers out
-        # with SQLITE_BUSY after their 50 ms busy_timeout. Splitting the
-        # read phase keeps the writer lock held only for the bulk
-        # INSERT/UPDATE pass.
         to_archive: list[dict] = []
         for row in rows:
             fid = row["fact_id"]
@@ -111,7 +108,7 @@ def run_reward_gated_archive(
             # 2. Recent positive reward skip (criterion 2).
             #    H-06 fix — JSON1 equality join via helper.
             if has_recent_positive_reward(
-                conn, profile_id, fid,
+                conn_r, profile_id, fid,
                 min_reward=ARCHIVE_REWARD_THRESHOLD,
                 window_days=REWARD_WINDOW_DAYS,
             ):
@@ -125,86 +122,97 @@ def run_reward_gated_archive(
                 "embedding": row["embedding"],
                 "created_at": row["created_at"],
             })
+    finally:
+        conn_r.close()
 
-        if not to_archive:
-            return []
-
-        conn.execute("BEGIN IMMEDIATE")
-        # S9-SKEP-07: re-verify reward under RESERVED lock. Between the
-        # read-only reward scan above and this BEGIN IMMEDIATE another
-        # writer may have inserted a positive reward row ("user liked
-        # the memory we are about to archive"). With the writer lock
-        # held we now know no further inserts are landing; one last
-        # check per entry catches everything that raced in.
-        verified: list[dict] = []
-        for entry in to_archive:
-            if has_recent_positive_reward(
-                conn, profile_id, entry["fid"],
-                min_reward=ARCHIVE_REWARD_THRESHOLD,
-                window_days=REWARD_WINDOW_DAYS,
-            ):
-                continue
-            verified.append(entry)
-        to_archive = verified
-        if not to_archive:
-            conn.execute("COMMIT")
-            return []
-        for entry in to_archive:
-            fid = entry["fid"]
-            payload = {
-                "fact_id": fid,
-                "content": entry["content"],
-                "canonical_entities_json": entry["canonical_entities_json"],
-                "importance": entry["importance"],
-                "confidence": entry["confidence"],
-                "embedding": entry["embedding"],
-                "created_at": entry["created_at"],
-            }
-            # SEC-L3 — cap payload_json at 256 KB. Oversize blobs are
-            # replaced with a minimal stub + ``truncated`` reason so the
-            # archive row stays within the I4 disk budget while still
-            # pointing back to the original ``fact_id`` in atomic_facts.
-            payload_str = json.dumps(payload)
-            reason = "reward_gated_ebbinghaus"
-            if len(payload_str.encode("utf-8")) > PAYLOAD_JSON_MAX_BYTES:
-                payload_str = json.dumps({
+    if not to_archive:
+        return []
+
+    # WRITE phase — acquire the process-level write lock BEFORE opening the
+    # write connection.  This serialises BEGIN IMMEDIATE → commit with all
+    # other in-process writers (DatabaseManager, VectorStore, adapter sync,
+    # memory_merge, reward_boost), eliminating SQLITE_BUSY races at the WAL
+    # layer.
+    with get_write_lock(memory_db_path):
+        conn = sqlite3.connect(str(memory_db_path), timeout=10.0)
+        conn.row_factory = sqlite3.Row
+        conn.execute("PRAGMA busy_timeout=2000")
+        try:
+            conn.execute("BEGIN IMMEDIATE")
+            # S9-SKEP-07: re-verify reward under RESERVED lock. Between the
+            # read-only reward scan above and this BEGIN IMMEDIATE another
+            # writer may have inserted a positive reward row. With the writer
+            # lock held we know no further inserts are landing; one last
+            # check per entry catches everything that raced in.
+            verified: list[dict] = []
+            for entry in to_archive:
+                if has_recent_positive_reward(
+                    conn, profile_id, entry["fid"],
+                    min_reward=ARCHIVE_REWARD_THRESHOLD,
+                    window_days=REWARD_WINDOW_DAYS,
+                ):
+                    continue
+                verified.append(entry)
+            to_archive = verified
+            if not to_archive:
+                conn.execute("COMMIT")
+                return []
+            for entry in to_archive:
+                fid = entry["fid"]
+                payload = {
                     "fact_id": fid,
-                    "truncated": True,
-                    "original_bytes": len(payload_str.encode("utf-8")),
-                })
-                reason = "reward_gated_ebbinghaus_truncated"
-                logger.warning(
-                    "memory_archive payload >%d bytes for fact_id=%s; "
-                    "truncated to stub", PAYLOAD_JSON_MAX_BYTES, fid,
+                    "content": entry["content"],
+                    "canonical_entities_json": entry["canonical_entities_json"],
+                    "importance": entry["importance"],
+                    "confidence": entry["confidence"],
+                    "embedding": entry["embedding"],
+                    "created_at": entry["created_at"],
+                }
+                # SEC-L3 — cap payload_json at 256 KB. Oversize blobs are
+                # replaced with a minimal stub + ``truncated`` reason so the
+                # archive row stays within the I4 disk budget while still
+                # pointing back to the original ``fact_id`` in atomic_facts.
+                payload_str = json.dumps(payload)
+                reason = "reward_gated_ebbinghaus"
+                if len(payload_str.encode("utf-8")) > PAYLOAD_JSON_MAX_BYTES:
+                    payload_str = json.dumps({
+                        "fact_id": fid,
+                        "truncated": True,
+                        "original_bytes": len(payload_str.encode("utf-8")),
+                    })
+                    reason = "reward_gated_ebbinghaus_truncated"
+                    logger.warning(
+                        "memory_archive payload >%d bytes for fact_id=%s; "
+                        "truncated to stub", PAYLOAD_JSON_MAX_BYTES, fid,
+                    )
+                conn.execute(
+                    "INSERT INTO memory_archive "
+                    "(archive_id, fact_id, profile_id, payload_json, "
+                    " archived_at, reason) VALUES (?, ?, ?, ?, ?, ?)",
+                    (
+                        str(uuid.uuid4()),
+                        fid,
+                        profile_id,
+                        payload_str,
+                        _iso_now(),
+                        reason,
+                    ),
                 )
-            conn.execute(
-                "INSERT INTO memory_archive "
-                "(archive_id, fact_id, profile_id, payload_json, "
-                " archived_at, reason) VALUES (?, ?, ?, ?, ?, ?)",
-                (
-                    str(uuid.uuid4()),
-                    fid,
-                    profile_id,
-                    payload_str,
-                    _iso_now(),
-                    reason,
-                ),
-            )
-            conn.execute(
-                "UPDATE atomic_facts "
-                "SET archive_status='archived', "
-                "    archive_reason='reward_gated_ebbinghaus' "
-                "WHERE fact_id=? "
-                "  AND (archive_status IS NULL OR archive_status='live')",
-                (fid,),
-            )
-            archived.append(fid)
-
-        conn.commit()
-    except sqlite3.Error as exc:
-        conn.rollback()
-        logger.warning("run_reward_gated_archive rollback: %s", exc)
-    finally:
-        conn.close()
+                conn.execute(
+                    "UPDATE atomic_facts "
+                    "SET archive_status='archived', "
+                    "    archive_reason='reward_gated_ebbinghaus' "
+                    "WHERE fact_id=? "
+                    "  AND (archive_status IS NULL OR archive_status='live')",
+                    (fid,),
+                )
+                archived.append(fid)
+
+            conn.commit()
+        except sqlite3.Error as exc:
+            conn.rollback()
+            logger.warning("run_reward_gated_archive rollback: %s", exc)
+        finally:
+            conn.close()
 
     return archived

package/src/superlocalmemory/learning/reward_boost.py

@@ -23,6 +23,7 @@ from pathlib import Path
 from superlocalmemory.learning.fact_outcome_joins import (
     aggregate_reward_for_fact,
 )
+from superlocalmemory.storage.write_lock import get_write_lock
 
 
 # H-12/H-P-01: single-pass JSON1 aggregation across ALL facts for a profile.
@@ -123,45 +124,54 @@ def apply_strong_memory_boost(
 
     Returns number of rows boosted.
     """
-    conn = sqlite3.connect(str(memory_db_path), timeout=10.0)
-    conn.execute("PRAGMA busy_timeout=2000")
-    boosted = 0
+    # READ phase — no write lock needed (WAL allows concurrent reads).
+    conn_r = sqlite3.connect(str(memory_db_path), timeout=10.0)
     try:
-        rows = conn.execute(
+        rows = conn_r.execute(
             "SELECT fact_id FROM atomic_facts WHERE profile_id=? "
             "  AND (archive_status IS NULL OR archive_status='live')",
             (profile_id,),
         ).fetchall()
         if not rows:
+            conn_r.close()
             return 0
-
         # H-12/H-P-01: single JSON1 GROUP BY replaces the per-fact loop.
-        # Fallback to per-fact helper preserves legacy behaviour on
-        # SQLite without JSON1.
-        stats = _bulk_fact_reward_stats(conn, profile_id)
-        conn.execute("BEGIN IMMEDIATE")
-        for (fid,) in rows:
-            if stats:
-                count, mean = stats.get(fid, _MISS)
-            else:
-                count, mean = aggregate_reward_for_fact(conn, profile_id, fid)
-            if count < STRONG_BOOST_MIN_OUTCOMES:
-                continue
-            if mean <= STRONG_BOOST_MIN_MEAN:
-                continue
-            conn.execute(
-                "UPDATE atomic_facts "
-                "SET retrieval_prior = MIN(COALESCE(retrieval_prior, 0) + ?, ?) "
-                "WHERE fact_id=?",
-                (STRONG_BOOST_INCREMENT, STRONG_BOOST_CAP, fid),
-            )
-            boosted += 1
-        conn.commit()
-    except sqlite3.Error as exc:
-        conn.rollback()
-        logger.warning("apply_strong_memory_boost rollback: %s", exc)
+        stats = _bulk_fact_reward_stats(conn_r, profile_id)
     finally:
-        conn.close()
+        conn_r.close()
+
+    # WRITE phase — acquire the process-level write lock BEFORE opening the
+    # write connection.  This serialises BEGIN IMMEDIATE → commit with all
+    # other in-process writers (DatabaseManager, VectorStore, adapter sync),
+    # eliminating SQLITE_BUSY races at the SQLite WAL layer.
+    boosted = 0
+    with get_write_lock(memory_db_path):
+        conn = sqlite3.connect(str(memory_db_path), timeout=10.0)
+        conn.execute("PRAGMA busy_timeout=2000")
+        try:
+            conn.execute("BEGIN IMMEDIATE")
+            for (fid,) in rows:
+                if stats:
+                    count, mean = stats.get(fid, _MISS)
+                else:
+                    count, mean = aggregate_reward_for_fact(conn, profile_id, fid)
+                if count < STRONG_BOOST_MIN_OUTCOMES:
+                    continue
+                if mean <= STRONG_BOOST_MIN_MEAN:
+                    continue
+                conn.execute(
+                    "UPDATE atomic_facts "
+                    "SET retrieval_prior = MIN(COALESCE(retrieval_prior, 0) + ?, ?) "
+                    "WHERE fact_id=?",
+                    (STRONG_BOOST_INCREMENT, STRONG_BOOST_CAP, fid),
+                )
+                boosted += 1
+            conn.commit()
+        except sqlite3.Error as exc:
+            conn.rollback()
+            logger.warning("apply_strong_memory_boost rollback: %s", exc)
+        finally:
+            conn.close()
     return boosted