superlocalmemory 3.8.3 → 3.8.6

package/src/superlocalmemory/infra/cloud_backup.py

@@ -26,6 +26,7 @@ from pathlib import Path
 from typing import Any
 
 from superlocalmemory.infra.data_root import canonical_data_root
+from superlocalmemory.storage.memory_write import memory_write
 
 logger = logging.getLogger("superlocalmemory.cloud_backup")
 
@@ -248,8 +249,7 @@ def add_destination(
 
     dest_id = _new_id()
     path = db_path or _default_db_path()
-    conn = sqlite3.connect(str(path))
-    try:
+    with memory_write(path) as conn:
         conn.execute(
             "INSERT INTO backup_destinations "
             "(id, destination_type, display_name, credentials_ref, config, "
@@ -257,32 +257,34 @@ def add_destination(
             (dest_id, destination_type, display_name, credentials_ref,
              json.dumps(config), datetime.now(UTC).isoformat()),
         )
-        conn.commit()
-        logger.info("Added backup destination: %s (%s)", display_name, destination_type)
-        return dest_id
-    finally:
-        conn.close()
+    logger.info("Added backup destination: %s (%s)", display_name, destination_type)
+    return dest_id
 
 
 def remove_destination(dest_id: str, db_path: Path | None = None) -> bool:
     """Remove a backup destination and its credentials."""
     path = db_path or _default_db_path()
-    conn = sqlite3.connect(str(path))
     try:
-        row = conn.execute(
-            "SELECT credentials_ref FROM backup_destinations WHERE id = ?",
-            (dest_id,),
-        ).fetchone()
-        if row and row[0]:
-            _delete_credential(row[0])
-        conn.execute("DELETE FROM backup_destinations WHERE id = ?", (dest_id,))
-        conn.commit()
+        # Fetch credentials_ref with a short read before taking the write lock.
+        import sqlite3 as _sqlite3
+        creds_ref = None
+        try:
+            with _sqlite3.connect(str(path)) as rconn:
+                row = rconn.execute(
+                    "SELECT credentials_ref FROM backup_destinations WHERE id = ?",
+                    (dest_id,),
+                ).fetchone()
+                creds_ref = row[0] if row else None
+        except Exception:
+            pass
+        if creds_ref:
+            _delete_credential(creds_ref)
+        with memory_write(path) as conn:
+            conn.execute("DELETE FROM backup_destinations WHERE id = ?", (dest_id,))
         return True
     except Exception as exc:
         logger.error("Failed to remove destination %s: %s", dest_id, exc)
         return False
-    finally:
-        conn.close()
 
 
 def update_sync_status(
@@ -293,18 +295,15 @@ def update_sync_status(
 ) -> None:
     """Update the sync status of a destination."""
     path = db_path or _default_db_path()
-    conn = sqlite3.connect(str(path))
     try:
-        conn.execute(
-            "UPDATE backup_destinations SET last_sync_at = ?, "
-            "last_sync_status = ?, last_sync_error = ? WHERE id = ?",
-            (datetime.now(UTC).isoformat(), status, error, dest_id),
-        )
-        conn.commit()
+        with memory_write(path) as conn:
+            conn.execute(
+                "UPDATE backup_destinations SET last_sync_at = ?, "
+                "last_sync_status = ?, last_sync_error = ? WHERE id = ?",
+                (datetime.now(UTC).isoformat(), status, error, dest_id),
+            )
     except Exception:
         pass
-    finally:
-        conn.close()
 
 
 # ---------------------------------------------------------------------------

package/src/superlocalmemory/infra/event_bus.py

@@ -14,9 +14,13 @@ import threading
 from collections import deque
 from datetime import datetime, timedelta, timezone
 from pathlib import Path
-from typing import Any, Callable, Dict, List, Optional
+from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional
 
 from superlocalmemory.infra.data_root import state_path
+from superlocalmemory.storage.write_lock import get_write_lock
+
+if TYPE_CHECKING:
+    from superlocalmemory.storage.database import DatabaseManager
 
 logger = logging.getLogger("superlocalmemory.events")
 
@@ -65,15 +69,25 @@ class EventBus:
     _instances_lock = threading.Lock()
 
     @classmethod
-    def get_instance(cls, db_path: Optional[Path] = None) -> "EventBus":
-        """Get or create the singleton EventBus for a database path."""
+    def get_instance(
+        cls,
+        db_path: Optional[Path] = None,
+        db: "DatabaseManager | None" = None,
+    ) -> "EventBus":
+        """Get or create the singleton EventBus for a database path.
+
+        Fix C: optional ``db`` forwarded to the constructor on first creation.
+        On subsequent calls for the same path the existing instance is returned;
+        callers that need to wire a DatabaseManager after construction should
+        call ``instance.set_db(db)`` separately.
+        """
         if db_path is None:
             db_path = state_path("memory.db")
 
         key = str(db_path)
         with cls._instances_lock:
             if key not in cls._instances:
-                cls._instances[key] = cls(db_path)
+                cls._instances[key] = cls(db_path, db=db)
             return cls._instances[key]
 
     @classmethod
@@ -87,9 +101,31 @@ class EventBus:
                 if key in cls._instances:
                     del cls._instances[key]
 
-    def __init__(self, db_path: Path) -> None:
-        """Initialize EventBus. Prefer get_instance() over direct construction."""
+    def __init__(
+        self,
+        db_path: Path,
+        db: "DatabaseManager | None" = None,
+    ) -> None:
+        """Initialize EventBus. Prefer get_instance() over direct construction.
+
+        Fix C: optional ``db`` parameter.  When provided, all write paths
+        (INSERT / UPDATE / DELETE) are routed through DatabaseManager.execute()
+        which acquires the process-level RLock before opening a connection.
+        This eliminates the EventBus write-storm contribution to the memory.db
+        lock contention: instead of 5 independent bare sqlite3.connect() calls
+        racing the materialiser, all writes queue through the single writer.
+
+        When ``db`` is None the fallback path is used — direct sqlite3.connect()
+        with ``PRAGMA busy_timeout=10000`` on every connection.  This is safe
+        for standalone / testing use but is NOT process-lock-serialised.
+        """
         self.db_path = Path(db_path)
+        self._db: "DatabaseManager | None" = db
+        if db is None:
+            logger.debug(
+                "EventBus operating in standalone mode — no DatabaseManager "
+                "provided; concurrency is unmanaged (busy_timeout=10000 applied)"
+            )
         self._buffer: deque = deque(maxlen=EVENT_BUFFER_SIZE)
         self._buffer_lock = threading.Lock()
         self._event_counter = 0
@@ -101,6 +137,15 @@ class EventBus:
         self._init_schema()
         logger.info("EventBus initialized: db=%s", self.db_path)
 
+    def set_db(self, db: "DatabaseManager") -> None:
+        """Wire EventBus to an already-initialised DatabaseManager.
+
+        For daemon start-up sequences where EventBus is constructed before
+        DatabaseManager is ready.  Call once from the daemon after both
+        are initialised.  Subsequent write calls will use ``db``.
+        """
+        self._db = db
+
     def _init_schema(self) -> None:
         """Create the memory_events table if it does not exist.
 
@@ -108,38 +153,46 @@ class EventBus:
         dashboard viewing profile A never sees profile B's events. This table is
         store-owned (created here, not by the migration runner), so the store
         owns its upgrade. Existing rows backfill to the 'default' profile.
+
+        Uses a direct connection here (before DatabaseManager may be available)
+        with busy_timeout=10000 for robustness during daemon start-up.
         """
-        conn = sqlite3.connect(str(self.db_path))
-        try:
-            cur = conn.cursor()
-            cur.execute("""
-                CREATE TABLE IF NOT EXISTS memory_events (
-                    id INTEGER PRIMARY KEY AUTOINCREMENT,
-                    profile_id TEXT NOT NULL DEFAULT 'default',
-                    event_type TEXT NOT NULL,
-                    memory_id INTEGER,
-                    source_agent TEXT DEFAULT 'user',
-                    source_protocol TEXT DEFAULT 'internal',
-                    payload TEXT,
-                    importance INTEGER DEFAULT 5,
-                    tier TEXT DEFAULT 'hot',
-                    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
-                )
-            """)
-            existing = {r[1] for r in cur.execute(
-                "PRAGMA table_info(memory_events)").fetchall()}
-            if "profile_id" not in existing:
-                cur.execute(
-                    "ALTER TABLE memory_events "
-                    "ADD COLUMN profile_id TEXT NOT NULL DEFAULT 'default'"
-                )
-            cur.execute("CREATE INDEX IF NOT EXISTS idx_events_type ON memory_events(event_type)")
-            cur.execute("CREATE INDEX IF NOT EXISTS idx_events_created ON memory_events(created_at)")
-            cur.execute("CREATE INDEX IF NOT EXISTS idx_events_tier ON memory_events(tier)")
-            cur.execute("CREATE INDEX IF NOT EXISTS idx_events_profile ON memory_events(profile_id, id)")
-            conn.commit()
-        finally:
-            conn.close()
+        # Startup DDL is a memory.db write — serialise it with the process
+        # write lock so a bus created concurrently with active writers cannot
+        # race the schema create/migrate at the WAL layer.
+        with get_write_lock(self.db_path):
+            conn = sqlite3.connect(str(self.db_path))
+            conn.execute("PRAGMA busy_timeout=10000")
+            try:
+                cur = conn.cursor()
+                cur.execute("""
+                    CREATE TABLE IF NOT EXISTS memory_events (
+                        id INTEGER PRIMARY KEY AUTOINCREMENT,
+                        profile_id TEXT NOT NULL DEFAULT 'default',
+                        event_type TEXT NOT NULL,
+                        memory_id INTEGER,
+                        source_agent TEXT DEFAULT 'user',
+                        source_protocol TEXT DEFAULT 'internal',
+                        payload TEXT,
+                        importance INTEGER DEFAULT 5,
+                        tier TEXT DEFAULT 'hot',
+                        created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+                    )
+                """)
+                existing = {r[1] for r in cur.execute(
+                    "PRAGMA table_info(memory_events)").fetchall()}
+                if "profile_id" not in existing:
+                    cur.execute(
+                        "ALTER TABLE memory_events "
+                        "ADD COLUMN profile_id TEXT NOT NULL DEFAULT 'default'"
+                    )
+                cur.execute("CREATE INDEX IF NOT EXISTS idx_events_type ON memory_events(event_type)")
+                cur.execute("CREATE INDEX IF NOT EXISTS idx_events_created ON memory_events(created_at)")
+                cur.execute("CREATE INDEX IF NOT EXISTS idx_events_tier ON memory_events(tier)")
+                cur.execute("CREATE INDEX IF NOT EXISTS idx_events_profile ON memory_events(profile_id, id)")
+                conn.commit()
+            finally:
+                conn.close()
 
     @staticmethod
     def _resolve_profile(profile_id: Optional[str]) -> str:
@@ -246,26 +299,60 @@ class EventBus:
     publish = emit
 
     def _persist_event(self, event: dict) -> Optional[int]:
-        """Persist event to the memory_events table. Returns row id or None."""
+        """Persist event to the memory_events table. Returns row id or None.
+
+        Fix C: when a DatabaseManager is available, routes the INSERT through
+        db.execute() (which acquires the process-level RLock for the write).
+        The fallback path (no db) uses a direct connection with
+        busy_timeout=10000 for resilience against short lock holds.
+
+        Note: db.execute() returns list[sqlite3.Row] — for an INSERT the
+        return value is empty.  The ROWID of the inserted row is retrieved
+        with a follow-up SELECT last_insert_rowid() on the same connection.
+        Because db.execute() is a per-call connect/commit/close, the rowid
+        is obtained inside the same logical operation.
+        """
+        sql = (
+            "INSERT INTO memory_events (profile_id, event_type, memory_id,"
+            " source_agent, source_protocol, payload, importance, tier,"
+            " created_at)"
+            " VALUES (?, ?, ?, ?, ?, ?, ?, 'hot', ?)"
+        )
+        params = (
+            event.get("profile_id", "default"),
+            event["event_type"],
+            event.get("memory_id"),
+            event["source_agent"],
+            event["source_protocol"],
+            json.dumps(event["payload"]),
+            event["importance"],
+            event["timestamp"],
+        )
+
         try:
-            conn = sqlite3.connect(str(self.db_path))
-            try:
-                cur = conn.cursor()
-                cur.execute(
-                    "INSERT INTO memory_events (profile_id, event_type, memory_id,"
-                    " source_agent, source_protocol, payload, importance, tier,"
-                    " created_at)"
-                    " VALUES (?, ?, ?, ?, ?, ?, ?, 'hot', ?)",
-                    (event.get("profile_id", "default"), event["event_type"],
-                     event.get("memory_id"),
-                     event["source_agent"], event["source_protocol"],
-                     json.dumps(event["payload"]), event["importance"],
-                     event["timestamp"]),
-                )
-                conn.commit()
-                return cur.lastrowid
-            finally:
-                conn.close()
+            if self._db is not None:
+                # Serialised path: routes through DatabaseManager's RLock.
+                # Use a transaction() context to get the lastrowid.
+                with self._db.transaction():
+                    self._db.execute(sql, params)
+                    rows = self._db.execute("SELECT last_insert_rowid() AS id")
+                    return int(rows[0]["id"]) if rows else None
+            else:
+                # Fallback: acquire the process write lock BEFORE opening the
+                # connection so this INSERT is serialised with every other
+                # memory.db writer even when no DatabaseManager was wired in
+                # (daemon/route/MCP callers construct via get_instance(path)).
+                # The write is a single tiny row — the lock is held for <1ms.
+                with get_write_lock(self.db_path):
+                    conn = sqlite3.connect(str(self.db_path))
+                    conn.execute("PRAGMA busy_timeout=10000")
+                    try:
+                        cur = conn.cursor()
+                        cur.execute(sql, params)
+                        conn.commit()
+                        return cur.lastrowid
+                    finally:
+                        conn.close()
         except Exception as exc:
             logger.error("Failed to persist event: %s", exc)
             return None
@@ -318,6 +405,7 @@ class EventBus:
 
         try:
             conn = sqlite3.connect(str(self.db_path))
+            conn.execute("PRAGMA busy_timeout=10000")
             try:
                 cur = conn.cursor()
 
@@ -379,6 +467,7 @@ class EventBus:
         scope = self._resolve_profile(profile_id)
         try:
             conn = sqlite3.connect(str(self.db_path))
+            conn.execute("PRAGMA busy_timeout=10000")
             try:
                 cur = conn.cursor()
 
@@ -427,42 +516,115 @@ class EventBus:
         surface. Tenant isolation of event CONTENT is enforced on read via the
         profile_id filter; this sweep only demotes/expires old rows by age.
         """
-        try:
-            conn = sqlite3.connect(str(self.db_path))
-            try:
-                cur = conn.cursor()
-                now = datetime.now()
-                stats = {"hot_to_warm": 0, "warm_to_cold": 0, "archived": 0}
-
-                # Hot -> Warm: older than hot_hours, importance < 5
-                warm_cutoff = (now - timedelta(hours=hot_hours)).isoformat()
-                cur.execute(
-                    "UPDATE memory_events SET tier = 'warm' "
-                    "WHERE tier = 'hot' AND created_at < ? AND importance < 5",
-                    (warm_cutoff,),
-                )
-                stats["hot_to_warm"] = cur.rowcount
+        now = datetime.now()
+        warm_cutoff = (now - timedelta(hours=hot_hours)).isoformat()
+        cold_cutoff = (now - timedelta(hours=warm_hours)).isoformat()
+        archive_cutoff = (now - timedelta(hours=cold_hours)).isoformat()
+        stats = {"hot_to_warm": 0, "warm_to_cold": 0, "archived": 0}
 
-                # Warm -> Cold: delete warm events older than warm_hours
-                cold_cutoff = (now - timedelta(hours=warm_hours)).isoformat()
-                cur.execute(
-                    "DELETE FROM memory_events "
-                    "WHERE tier = 'warm' AND created_at < ?",
-                    (cold_cutoff,),
-                )
-                stats["warm_to_cold"] = cur.rowcount
+        _PRUNE_BATCH = 5000  # max rows per transaction — keeps lock time bounded
 
-                # Archive: delete everything older than cold_hours
-                archive_cutoff = (now - timedelta(hours=cold_hours)).isoformat()
-                cur.execute(
-                    "DELETE FROM memory_events WHERE created_at < ?",
-                    (archive_cutoff,),
+        if self._db is not None:
+            # Fix C: route tier-management writes through DatabaseManager.
+            # Use separate short transactions per batch to avoid one long hold.
+            try:
+                # Hot → warm (UPDATE, bounded loop)
+                while True:
+                    with self._db.transaction():
+                        self._db.execute(
+                            "UPDATE memory_events SET tier = 'warm' "
+                            "WHERE rowid IN ("
+                            "  SELECT rowid FROM memory_events "
+                            "  WHERE tier = 'hot' AND created_at < ? AND importance < 5 "
+                            "  LIMIT ?"
+                            ")",
+                            (warm_cutoff, _PRUNE_BATCH),
+                        )
+                        rows = self._db.execute("SELECT changes()")
+                        changed = rows[0][0] if rows else 0
+                    stats["hot_to_warm"] += changed
+                    if changed < _PRUNE_BATCH:
+                        break
+
+                # Warm → cold (DELETE, bounded loop)
+                while True:
+                    with self._db.transaction():
+                        self._db.execute(
+                            "DELETE FROM memory_events "
+                            "WHERE rowid IN ("
+                            "  SELECT rowid FROM memory_events "
+                            "  WHERE tier = 'warm' AND created_at < ? "
+                            "  LIMIT ?"
+                            ")",
+                            (cold_cutoff, _PRUNE_BATCH),
+                        )
+                        rows = self._db.execute("SELECT changes()")
+                        changed = rows[0][0] if rows else 0
+                    stats["warm_to_cold"] += changed
+                    if changed < _PRUNE_BATCH:
+                        break
+
+                # Archive prune (DELETE, bounded loop)
+                while True:
+                    with self._db.transaction():
+                        self._db.execute(
+                            "DELETE FROM memory_events "
+                            "WHERE rowid IN ("
+                            "  SELECT rowid FROM memory_events "
+                            "  WHERE created_at < ? "
+                            "  LIMIT ?"
+                            ")",
+                            (archive_cutoff, _PRUNE_BATCH),
+                        )
+                        rows = self._db.execute("SELECT changes()")
+                        changed = rows[0][0] if rows else 0
+                    stats["archived"] += changed
+                    if changed < _PRUNE_BATCH:
+                        break
+
+                logger.info(
+                    "Prune complete (via db): hot->warm=%d warm->cold=%d archived=%d",
+                    stats["hot_to_warm"], stats["warm_to_cold"], stats["archived"],
                 )
-                stats["archived"] = cur.rowcount
+                return stats
+            except Exception as exc:
+                logger.error("Event pruning failed (db path): %s", exc)
+                return {"error": str(exc)}
 
-                conn.commit()
-            finally:
-                conn.close()
+        # Fallback: acquire the process write lock so this maintenance prune
+        # is serialised with all other memory.db writers even when no
+        # DatabaseManager was wired in. memory_events is a small bounded table
+        # (retention-capped) so the lock is held only briefly.
+        try:
+            with get_write_lock(self.db_path):
+                conn = sqlite3.connect(str(self.db_path))
+                conn.execute("PRAGMA busy_timeout=10000")
+                try:
+                    cur = conn.cursor()
+
+                    cur.execute(
+                        "UPDATE memory_events SET tier = 'warm' "
+                        "WHERE tier = 'hot' AND created_at < ? AND importance < 5",
+                        (warm_cutoff,),
+                    )
+                    stats["hot_to_warm"] = cur.rowcount
+
+                    cur.execute(
+                        "DELETE FROM memory_events "
+                        "WHERE tier = 'warm' AND created_at < ?",
+                        (cold_cutoff,),
+                    )
+                    stats["warm_to_cold"] = cur.rowcount
+
+                    cur.execute(
+                        "DELETE FROM memory_events WHERE created_at < ?",
+                        (archive_cutoff,),
+                    )
+                    stats["archived"] = cur.rowcount
+
+                    conn.commit()
+                finally:
+                    conn.close()
 
             logger.info(
                 "Prune complete: hot->warm=%d warm->cold=%d archived=%d",

package/src/superlocalmemory/learning/bandit.py

@@ -29,7 +29,6 @@ import os
 import secrets
 import sqlite3
 import threading
-import time
 from dataclasses import dataclass, field
 from datetime import datetime, timedelta, timezone
 from pathlib import Path
@@ -262,6 +261,33 @@ class ContextualBandit:
             play_id=play_id,
         )
 
+    def choose_readonly(self, context: dict[str, Any]) -> BanditChoice:
+        """Sample an arm from a read-only snapshot without recording a play.
+
+        Recall uses this method so the established bandit weighting and
+        ensemble quality path remain available without turning a query into a
+        ``bandit_plays`` write.  It deliberately bypasses the writer-oriented
+        thread-local connection factory: that factory configures WAL mode and
+        is not a physical read-only guarantee.
+        """
+        stratum = compute_stratum(context)
+        try:
+            posteriors = self._load_stratum_posteriors_readonly(stratum)
+        except sqlite3.Error as exc:
+            logger.warning(
+                "bandit.choose_readonly: posterior load failed stratum=%s: %s",
+                stratum,
+                exc,
+            )
+            posteriors = {}
+        arm_id = self._sample_best(posteriors)
+        return BanditChoice(
+            stratum=stratum,
+            arm_id=arm_id,
+            weights=dict(self._catalog[arm_id]),
+            play_id=None,
+        )
+
     def _sample_best(
         self,
         posteriors: dict[str, tuple[float, float]],
@@ -438,6 +464,29 @@ class ContextualBandit:
             for r in rows
         }
 
+    def _load_stratum_posteriors_readonly(
+        self,
+        stratum: str,
+    ) -> dict[str, tuple[float, float]]:
+        """Read posteriors through SQLite's URI read-only boundary."""
+        uri = f"{self._db_path.resolve().as_uri()}?mode=ro"
+        conn = sqlite3.connect(uri, uri=True, timeout=0.25)
+        try:
+            conn.row_factory = sqlite3.Row
+            conn.execute("PRAGMA query_only=ON")
+            conn.execute("PRAGMA busy_timeout=250")
+            rows = conn.execute(
+                "SELECT arm_id, alpha, beta FROM bandit_arms "
+                "WHERE profile_id = ? AND stratum = ?",
+                (self._profile, stratum),
+            ).fetchall()
+        finally:
+            conn.close()
+        return {
+            row["arm_id"]: (float(row["alpha"]), float(row["beta"]))
+            for row in rows
+        }
+
     def _insert_play(
         self,
         query_id: str,