superlocalmemory 4.1.13 → 4.1.14

package/src/superlocalmemory/infra/event_bus.py

@@ -1,637 +0,0 @@
-# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
-# Licensed under AGPL-3.0-or-later - see LICENSE file
-# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
-"""EventBus -- Real-time event broadcasting for memory operations.
-
-Thread-safe singleton per DB path. In-memory deque buffer + SQLite persistence.
-Listener callbacks run on the emitter's thread.
-"""
-
-import json
-import logging
-import sqlite3
-import threading
-from collections import deque
-from datetime import datetime, timedelta, timezone
-from pathlib import Path
-from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional
-
-from superlocalmemory.infra.data_root import state_path
-from superlocalmemory.storage.write_lock import get_write_lock
-
-if TYPE_CHECKING:
-    from superlocalmemory.storage.database import DatabaseManager
-
-logger = logging.getLogger("superlocalmemory.events")
-
-# Default retention windows (hours)
-DEFAULT_HOT_HOURS = 48
-DEFAULT_WARM_HOURS = 14 * 24  # 14 days
-DEFAULT_COLD_HOURS = 30 * 24  # 30 days
-
-# In-memory buffer size for real-time delivery
-EVENT_BUFFER_SIZE = 200
-
-# Valid event types (V3 superset)
-VALID_EVENT_TYPES = frozenset([
-    "memory.stored",        # New memory written (was memory.created in V2)
-    "memory.updated",       # Existing memory modified
-    "memory.deleted",       # Memory removed
-    "memory.recalled",      # Memory retrieved by an agent
-    "memory.observed",      # /observe accepted content into the debounce buffer
-    "memory.captured",      # AutoCapture matched a buffered observation
-    "memory.dropped",       # AutoCapture rejected a buffered observation
-    "memory.queued",        # /remember accepted content into pending.db (async)
-    "graph.updated",        # Knowledge graph rebuilt
-    "pattern.learned",      # New pattern detected
-    "agent.connected",      # New agent connects
-    "agent.disconnected",   # Agent disconnects
-    "trust.signal",         # V3: trust score change
-    "compliance.audit",     # V3: compliance event logged
-    "learning.feedback",    # V3: learning feedback received
-    # CodeGraph events (v3.4) — NOTE: "graph.updated" is SLM entity graph, "code_graph.*" is AST code graph
-    "code_graph.built",          # Full code graph build completed
-    "code_graph.updated",        # Incremental code graph update completed
-    "code_graph.node_changed",   # Function/class signature or body changed
-    "code_graph.node_deleted",   # Function/class/file removed from codebase
-])
-
-
-class EventBus:
-    """
-    Central event bus for SuperLocalMemory V3.
-
-    Singleton per database path. Emits events to persistent storage and
-    in-memory listeners simultaneously. Thread-safe.
-    """
-
-    _instances: Dict[str, "EventBus"] = {}
-    _instances_lock = threading.Lock()
-
-    @classmethod
-    def get_instance(
-        cls,
-        db_path: Optional[Path] = None,
-        db: "DatabaseManager | None" = None,
-    ) -> "EventBus":
-        """Get or create the singleton EventBus for a database path.
-
-        Fix C: optional ``db`` forwarded to the constructor on first creation.
-        On subsequent calls for the same path the existing instance is returned;
-        callers that need to wire a DatabaseManager after construction should
-        call ``instance.set_db(db)`` separately.
-        """
-        if db_path is None:
-            db_path = state_path("memory.db")
-
-        key = str(db_path)
-        with cls._instances_lock:
-            if key not in cls._instances:
-                cls._instances[key] = cls(db_path, db=db)
-            return cls._instances[key]
-
-    @classmethod
-    def reset_instance(cls, db_path: Optional[Path] = None) -> None:
-        """Remove and close a singleton instance. Used for testing."""
-        with cls._instances_lock:
-            if db_path is None:
-                cls._instances.clear()
-            else:
-                key = str(db_path)
-                if key in cls._instances:
-                    del cls._instances[key]
-
-    def __init__(
-        self,
-        db_path: Path,
-        db: "DatabaseManager | None" = None,
-    ) -> None:
-        """Initialize EventBus. Prefer get_instance() over direct construction.
-
-        Fix C: optional ``db`` parameter.  When provided, all write paths
-        (INSERT / UPDATE / DELETE) are routed through DatabaseManager.execute()
-        which acquires the process-level RLock before opening a connection.
-        This eliminates the EventBus write-storm contribution to the memory.db
-        lock contention: instead of 5 independent bare sqlite3.connect() calls
-        racing the materialiser, all writes queue through the single writer.
-
-        When ``db`` is None the fallback path is used — direct sqlite3.connect()
-        with ``PRAGMA busy_timeout=10000`` on every connection.  This is safe
-        for standalone / testing use but is NOT process-lock-serialised.
-        """
-        self.db_path = Path(db_path)
-        self._db: "DatabaseManager | None" = db
-        if db is None:
-            logger.debug(
-                "EventBus operating in standalone mode — no DatabaseManager "
-                "provided; concurrency is unmanaged (busy_timeout=10000 applied)"
-            )
-        self._buffer: deque = deque(maxlen=EVENT_BUFFER_SIZE)
-        self._buffer_lock = threading.Lock()
-        self._event_counter = 0
-        self._counter_lock = threading.Lock()
-        self._listeners: List[Callable[[dict], None]] = []
-        self._listeners_lock = threading.Lock()
-        self._write_count = 0
-        self._last_prune = datetime.now()
-        self._init_schema()
-        logger.info("EventBus initialized: db=%s", self.db_path)
-
-    def set_db(self, db: "DatabaseManager") -> None:
-        """Wire EventBus to an already-initialised DatabaseManager.
-
-        For daemon start-up sequences where EventBus is constructed before
-        DatabaseManager is ready.  Call once from the daemon after both
-        are initialised.  Subsequent write calls will use ``db``.
-        """
-        self._db = db
-
-    def _init_schema(self) -> None:
-        """Create the memory_events table if it does not exist.
-
-        Self-migrates a pre-isolation DB (memory_events without profile_id) so a
-        dashboard viewing profile A never sees profile B's events. This table is
-        store-owned (created here, not by the migration runner), so the store
-        owns its upgrade. Existing rows backfill to the 'default' profile.
-
-        Uses a direct connection here (before DatabaseManager may be available)
-        with busy_timeout=10000 for robustness during daemon start-up.
-        """
-        # Startup DDL is a memory.db write — serialise it with the process
-        # write lock so a bus created concurrently with active writers cannot
-        # race the schema create/migrate at the WAL layer.
-        with get_write_lock(self.db_path):
-            conn = sqlite3.connect(str(self.db_path))
-            conn.execute("PRAGMA busy_timeout=10000")
-            try:
-                cur = conn.cursor()
-                cur.execute("""
-                    CREATE TABLE IF NOT EXISTS memory_events (
-                        id INTEGER PRIMARY KEY AUTOINCREMENT,
-                        profile_id TEXT NOT NULL DEFAULT 'default',
-                        event_type TEXT NOT NULL,
-                        memory_id INTEGER,
-                        source_agent TEXT DEFAULT 'user',
-                        source_protocol TEXT DEFAULT 'internal',
-                        payload TEXT,
-                        importance INTEGER DEFAULT 5,
-                        tier TEXT DEFAULT 'hot',
-                        created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
-                    )
-                """)
-                existing = {r[1] for r in cur.execute(
-                    "PRAGMA table_info(memory_events)").fetchall()}
-                if "profile_id" not in existing:
-                    cur.execute(
-                        "ALTER TABLE memory_events "
-                        "ADD COLUMN profile_id TEXT NOT NULL DEFAULT 'default'"
-                    )
-                cur.execute("CREATE INDEX IF NOT EXISTS idx_events_type ON memory_events(event_type)")
-                cur.execute("CREATE INDEX IF NOT EXISTS idx_events_created ON memory_events(created_at)")
-                cur.execute("CREATE INDEX IF NOT EXISTS idx_events_tier ON memory_events(tier)")
-                cur.execute("CREATE INDEX IF NOT EXISTS idx_events_profile ON memory_events(profile_id, id)")
-                conn.commit()
-            finally:
-                conn.close()
-
-    @staticmethod
-    def _resolve_profile(profile_id: Optional[str]) -> str:
-        """Resolve the active profile for an event when not passed explicitly.
-
-        Uses the request-runtime helper (ContextVar for HTTP, else the
-        profiles.json active_profile cache that every switch keeps in sync) so
-        MCP-in-daemon and CLI emits are attributed correctly too. Lazy import
-        keeps the infra layer free of a hard server dependency; any failure
-        falls back to 'default'.
-        """
-        if profile_id:
-            return profile_id
-        try:
-            from superlocalmemory.server.routes.helpers import get_active_profile
-            resolved = get_active_profile()
-            if resolved:
-                return resolved
-        except Exception:
-            pass
-        return "default"
-
-    def emit(
-        self,
-        event_type: str,
-        payload: Optional[Dict[str, Any]] = None,
-        memory_id: Optional[int] = None,
-        source_agent: str = "user",
-        source_protocol: str = "internal",
-        importance: int = 5,
-        profile_id: Optional[str] = None,
-    ) -> Optional[int]:
-        """Emit an event to all subscribers and persist to database.
-
-        ``profile_id`` scopes the event to a memory profile. When omitted it is
-        resolved from the active profile so a dashboard viewing one profile
-        never sees another profile's real-time or historical events.
-        """
-        if event_type not in VALID_EVENT_TYPES:
-            raise ValueError(
-                f"Invalid event type: {event_type}. "
-                f"Valid: {', '.join(sorted(VALID_EVENT_TYPES))}"
-            )
-
-        importance = max(1, min(10, importance))
-        profile_id = self._resolve_profile(profile_id)
-
-        now = datetime.now(timezone.utc).isoformat()
-        with self._counter_lock:
-            self._event_counter += 1
-            seq = self._event_counter
-
-        event: Dict[str, Any] = {
-            "seq": seq,
-            "profile_id": profile_id,
-            "event_type": event_type,
-            "memory_id": memory_id,
-            "source_agent": source_agent,
-            "source_protocol": source_protocol,
-            "payload": payload or {},
-            "importance": importance,
-            "timestamp": now,
-        }
-
-        # 1. Persist
-        event_id = self._persist_event(event)
-        if event_id is not None:
-            event["id"] = event_id
-
-        # 2. Buffer
-        with self._buffer_lock:
-            self._buffer.append(event)
-
-        # 3. Notify
-        self._notify_listeners(event)
-
-        logger.debug(
-            "Event emitted: type=%s id=%s memory_id=%s",
-            event_type, event_id, memory_id,
-        )
-
-        # Auto-prune heuristic. Decide + reset the counter atomically under the
-        # lock so concurrent emit() calls cannot both cross the threshold and
-        # double-run the prune; run the prune itself OUTSIDE the lock.
-        should_prune = False
-        with self._counter_lock:
-            self._write_count += 1
-            if (
-                self._write_count >= 100
-                or (datetime.now() - self._last_prune).total_seconds() > 86400
-            ):
-                should_prune = True
-                self._write_count = 0
-                self._last_prune = datetime.now()
-        if should_prune:
-            try:
-                self.prune_events()
-            except Exception:
-                pass
-
-        return event_id
-
-    # Alias for V3 compatibility
-    publish = emit
-
-    def _persist_event(self, event: dict) -> Optional[int]:
-        """Persist event to the memory_events table. Returns row id or None.
-
-        Fix C: when a DatabaseManager is available, routes the INSERT through
-        db.execute() (which acquires the process-level RLock for the write).
-        The fallback path (no db) uses a direct connection with
-        busy_timeout=10000 for resilience against short lock holds.
-
-        Note: db.execute() returns list[sqlite3.Row] — for an INSERT the
-        return value is empty.  The ROWID of the inserted row is retrieved
-        with a follow-up SELECT last_insert_rowid() on the same connection.
-        Because db.execute() is a per-call connect/commit/close, the rowid
-        is obtained inside the same logical operation.
-        """
-        sql = (
-            "INSERT INTO memory_events (profile_id, event_type, memory_id,"
-            " source_agent, source_protocol, payload, importance, tier,"
-            " created_at)"
-            " VALUES (?, ?, ?, ?, ?, ?, ?, 'hot', ?)"
-        )
-        params = (
-            event.get("profile_id", "default"),
-            event["event_type"],
-            event.get("memory_id"),
-            event["source_agent"],
-            event["source_protocol"],
-            json.dumps(event["payload"]),
-            event["importance"],
-            event["timestamp"],
-        )
-
-        try:
-            if self._db is not None:
-                # Serialised path: routes through DatabaseManager's RLock.
-                # Use a transaction() context to get the lastrowid.
-                with self._db.transaction():
-                    self._db.execute(sql, params)
-                    rows = self._db.execute("SELECT last_insert_rowid() AS id")
-                    return int(rows[0]["id"]) if rows else None
-            else:
-                # Fallback: acquire the process write lock BEFORE opening the
-                # connection so this INSERT is serialised with every other
-                # memory.db writer even when no DatabaseManager was wired in
-                # (daemon/route/MCP callers construct via get_instance(path)).
-                # The write is a single tiny row — the lock is held for <1ms.
-                with get_write_lock(self.db_path):
-                    conn = sqlite3.connect(str(self.db_path))
-                    conn.execute("PRAGMA busy_timeout=10000")
-                    try:
-                        cur = conn.cursor()
-                        cur.execute(sql, params)
-                        conn.commit()
-                        return cur.lastrowid
-                    finally:
-                        conn.close()
-        except Exception as exc:
-            logger.error("Failed to persist event: %s", exc)
-            return None
-
-    def add_listener(self, callback: Callable[[dict], None]) -> None:
-        """Register a listener that receives every emitted event."""
-        with self._listeners_lock:
-            self._listeners.append(callback)
-
-    # Alias
-    subscribe = add_listener
-
-    def remove_listener(self, callback: Callable[[dict], None]) -> None:
-        """Remove a previously registered listener."""
-        with self._listeners_lock:
-            try:
-                self._listeners.remove(callback)
-            except ValueError:
-                pass
-
-    # Alias
-    unsubscribe = remove_listener
-
-    def _notify_listeners(self, event: dict) -> None:
-        """Call all registered listeners. Errors are logged, not raised."""
-        with self._listeners_lock:
-            listeners = list(self._listeners)
-
-        for listener in listeners:
-            try:
-                listener(event)
-            except Exception as exc:
-                logger.error("Event listener failed: %s", exc)
-
-    def get_recent_events(
-        self,
-        since_id: Optional[int] = None,
-        limit: int = 50,
-        event_type: Optional[str] = None,
-        profile_id: Optional[str] = None,
-    ) -> List[dict]:
-        """Get recent events from the database, scoped to a profile.
-
-        ``profile_id`` defaults to the active profile so callers never leak
-        another profile's events. Pass ``profile_id="*"`` to bypass scoping
-        (internal maintenance/pruning only — never a client-facing path).
-        """
-        limit = min(limit, 200)
-        scope = profile_id if profile_id == "*" else self._resolve_profile(profile_id)
-
-        try:
-            conn = sqlite3.connect(str(self.db_path))
-            conn.execute("PRAGMA busy_timeout=10000")
-            try:
-                cur = conn.cursor()
-
-                query = ("SELECT id, event_type, memory_id, source_agent,"
-                         " source_protocol, payload, importance, tier, created_at,"
-                         " profile_id"
-                         " FROM memory_events WHERE 1=1")
-                params: List[Any] = []
-
-                if scope != "*":
-                    query += " AND profile_id = ?"
-                    params.append(scope)
-
-                if since_id is not None:
-                    query += " AND id > ?"
-                    params.append(since_id)
-
-                if event_type:
-                    query += " AND event_type = ?"
-                    params.append(event_type)
-
-                query += " ORDER BY id ASC LIMIT ?"
-                params.append(limit)
-
-                cur.execute(query, params)
-                rows = cur.fetchall()
-            finally:
-                conn.close()
-
-            events: List[dict] = []
-            for row in rows:
-                try:
-                    parsed = json.loads(row[5]) if row[5] else {}
-                except (json.JSONDecodeError, TypeError):
-                    parsed = {}
-                events.append({
-                    "id": row[0], "event_type": row[1], "memory_id": row[2],
-                    "source_agent": row[3], "source_protocol": row[4],
-                    "payload": parsed, "importance": row[6],
-                    "tier": row[7], "timestamp": row[8], "profile_id": row[9],
-                })
-            return events
-
-        except Exception as exc:
-            logger.error("Failed to get recent events: %s", exc)
-            return []
-
-    def get_buffered_events(self, since_seq: int = 0) -> List[dict]:
-        """Get events from the in-memory buffer (no DB hit)."""
-        with self._buffer_lock:
-            return [e for e in self._buffer if e.get("seq", 0) > since_seq]
-
-    def get_event_stats(self, profile_id: Optional[str] = None) -> dict:
-        """Get event system statistics, scoped to a profile.
-
-        ``profile_id`` defaults to the active profile so dashboard event
-        counts never blend across profiles.
-        """
-        scope = self._resolve_profile(profile_id)
-        try:
-            conn = sqlite3.connect(str(self.db_path))
-            conn.execute("PRAGMA busy_timeout=10000")
-            try:
-                cur = conn.cursor()
-
-                total = cur.execute(
-                    "SELECT COUNT(*) FROM memory_events WHERE profile_id = ?",
-                    (scope,)).fetchone()[0]
-                cur.execute(
-                    "SELECT event_type, COUNT(*) FROM memory_events "
-                    "WHERE profile_id = ? GROUP BY event_type", (scope,))
-                by_type = dict(cur.fetchall())
-                cur.execute(
-                    "SELECT tier, COUNT(*) FROM memory_events "
-                    "WHERE profile_id = ? GROUP BY tier", (scope,))
-                by_tier = dict(cur.fetchall())
-                cur.execute(
-                    "SELECT COUNT(*) FROM memory_events WHERE profile_id = ? "
-                    "AND created_at >= datetime('now', '-24 hours')", (scope,))
-                last_24h = cur.fetchone()[0]
-            finally:
-                conn.close()
-
-            return {
-                **by_type,
-                "total_events": total,
-                "events_last_24h": last_24h,
-                "by_type": by_type,
-                "by_tier": by_tier,
-                "buffer_size": len(self._buffer),
-                "listener_count": len(self._listeners),
-            }
-
-        except Exception as exc:
-            logger.error("Failed to get event stats: %s", exc)
-            return {"total_events": 0, "error": str(exc)}
-
-    def prune_events(
-        self,
-        hot_hours: int = DEFAULT_HOT_HOURS,
-        warm_hours: int = DEFAULT_WARM_HOURS,
-        cold_hours: int = DEFAULT_COLD_HOURS,
-    ) -> dict:
-        """Apply tiered retention policy to persisted events.
-
-        INTENTIONALLY GLOBAL (all profiles): this is age/tier-based housekeeping
-        of the shared event log, not a per-tenant read or retention-policy
-        surface. Tenant isolation of event CONTENT is enforced on read via the
-        profile_id filter; this sweep only demotes/expires old rows by age.
-        """
-        now = datetime.now()
-        warm_cutoff = (now - timedelta(hours=hot_hours)).isoformat()
-        cold_cutoff = (now - timedelta(hours=warm_hours)).isoformat()
-        archive_cutoff = (now - timedelta(hours=cold_hours)).isoformat()
-        stats = {"hot_to_warm": 0, "warm_to_cold": 0, "archived": 0}
-
-        _PRUNE_BATCH = 5000  # max rows per transaction — keeps lock time bounded
-
-        if self._db is not None:
-            # Fix C: route tier-management writes through DatabaseManager.
-            # Use separate short transactions per batch to avoid one long hold.
-            try:
-                # Hot → warm (UPDATE, bounded loop)
-                while True:
-                    with self._db.transaction():
-                        self._db.execute(
-                            "UPDATE memory_events SET tier = 'warm' "
-                            "WHERE rowid IN ("
-                            "  SELECT rowid FROM memory_events "
-                            "  WHERE tier = 'hot' AND created_at < ? AND importance < 5 "
-                            "  LIMIT ?"
-                            ")",
-                            (warm_cutoff, _PRUNE_BATCH),
-                        )
-                        rows = self._db.execute("SELECT changes()")
-                        changed = rows[0][0] if rows else 0
-                    stats["hot_to_warm"] += changed
-                    if changed < _PRUNE_BATCH:
-                        break
-
-                # Warm → cold (DELETE, bounded loop)
-                while True:
-                    with self._db.transaction():
-                        self._db.execute(
-                            "DELETE FROM memory_events "
-                            "WHERE rowid IN ("
-                            "  SELECT rowid FROM memory_events "
-                            "  WHERE tier = 'warm' AND created_at < ? "
-                            "  LIMIT ?"
-                            ")",
-                            (cold_cutoff, _PRUNE_BATCH),
-                        )
-                        rows = self._db.execute("SELECT changes()")
-                        changed = rows[0][0] if rows else 0
-                    stats["warm_to_cold"] += changed
-                    if changed < _PRUNE_BATCH:
-                        break
-
-                # Archive prune (DELETE, bounded loop)
-                while True:
-                    with self._db.transaction():
-                        self._db.execute(
-                            "DELETE FROM memory_events "
-                            "WHERE rowid IN ("
-                            "  SELECT rowid FROM memory_events "
-                            "  WHERE created_at < ? "
-                            "  LIMIT ?"
-                            ")",
-                            (archive_cutoff, _PRUNE_BATCH),
-                        )
-                        rows = self._db.execute("SELECT changes()")
-                        changed = rows[0][0] if rows else 0
-                    stats["archived"] += changed
-                    if changed < _PRUNE_BATCH:
-                        break
-
-                logger.info(
-                    "Prune complete (via db): hot->warm=%d warm->cold=%d archived=%d",
-                    stats["hot_to_warm"], stats["warm_to_cold"], stats["archived"],
-                )
-                return stats
-            except Exception as exc:
-                logger.error("Event pruning failed (db path): %s", exc)
-                return {"error": str(exc)}
-
-        # Fallback: acquire the process write lock so this maintenance prune
-        # is serialised with all other memory.db writers even when no
-        # DatabaseManager was wired in. memory_events is a small bounded table
-        # (retention-capped) so the lock is held only briefly.
-        try:
-            with get_write_lock(self.db_path):
-                conn = sqlite3.connect(str(self.db_path))
-                conn.execute("PRAGMA busy_timeout=10000")
-                try:
-                    cur = conn.cursor()
-
-                    cur.execute(
-                        "UPDATE memory_events SET tier = 'warm' "
-                        "WHERE tier = 'hot' AND created_at < ? AND importance < 5",
-                        (warm_cutoff,),
-                    )
-                    stats["hot_to_warm"] = cur.rowcount
-
-                    cur.execute(
-                        "DELETE FROM memory_events "
-                        "WHERE tier = 'warm' AND created_at < ?",
-                        (cold_cutoff,),
-                    )
-                    stats["warm_to_cold"] = cur.rowcount
-
-                    cur.execute(
-                        "DELETE FROM memory_events WHERE created_at < ?",
-                        (archive_cutoff,),
-                    )
-                    stats["archived"] = cur.rowcount
-
-                    conn.commit()
-                finally:
-                    conn.close()
-
-            logger.info(
-                "Prune complete: hot->warm=%d warm->cold=%d archived=%d",
-                stats["hot_to_warm"], stats["warm_to_cold"], stats["archived"],
-            )
-            return stats
-
-        except Exception as exc:
-            logger.error("Event pruning failed: %s", exc)
-            return {"error": str(exc)}