superlocalmemory 2.8.5 → 3.0.0

package/src/superlocalmemory/infra/cache_manager.py

@@ -0,0 +1,267 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under the MIT License - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+"""LRU cache with optional TTL for search-result caching.
+
+Key features:
+    * O(1) get / set via ``OrderedDict``
+    * Time-to-live expiry per entry
+    * Size-based eviction (LRU)
+    * Optional thread safety
+    * Hit / miss / eviction statistics
+"""
+
+import hashlib
+import json
+import logging
+import time
+from collections import OrderedDict
+from threading import RLock
+from typing import Any, Dict, List, Optional, Tuple
+
+logger = logging.getLogger("superlocalmemory.cache")
+
+
+class CacheEntry:
+    """Single cache entry with metadata."""
+
+    __slots__ = ["value", "timestamp", "access_count", "size_estimate"]
+
+    def __init__(self, value: Any, size_estimate: int = 0) -> None:
+        self.value = value
+        self.timestamp = time.time()
+        self.access_count = 0
+        self.size_estimate = size_estimate
+
+    def is_expired(self, ttl_seconds: Optional[float]) -> bool:
+        """Return ``True`` when the entry has exceeded *ttl_seconds*."""
+        if ttl_seconds is None:
+            return False
+        return (time.time() - self.timestamp) > ttl_seconds
+
+    def mark_accessed(self) -> None:
+        """Increment access counter."""
+        self.access_count += 1
+
+
+class CacheManager:
+    """LRU cache manager with TTL support.
+
+    Args:
+        max_size: Maximum number of entries before LRU eviction.
+        ttl_seconds: Per-entry time-to-live (``None`` = never expire).
+        thread_safe: Wrap operations in a reentrant lock.
+    """
+
+    def __init__(
+        self,
+        max_size: int = 100,
+        ttl_seconds: Optional[float] = 300,
+        thread_safe: bool = False,
+    ) -> None:
+        self.max_size = max(1, max_size)
+        self.ttl_seconds = ttl_seconds
+        self.thread_safe = thread_safe
+
+        self._cache: OrderedDict[str, CacheEntry] = OrderedDict()
+        self._lock: Optional[RLock] = RLock() if thread_safe else None
+
+        # Statistics
+        self._hits = 0
+        self._misses = 0
+        self._evictions = 0
+        self._total_size_estimate = 0
+
+    # ------------------------------------------------------------------
+    # Key helpers
+    # ------------------------------------------------------------------
+
+    @staticmethod
+    def _hash_key(query: str, **kwargs: Any) -> str:
+        """Deterministic cache key from *query* + keyword args."""
+        key_data = {"query": query, **kwargs}
+        key_str = json.dumps(key_data, sort_keys=True)
+        return hashlib.sha256(key_str.encode()).hexdigest()[:16]
+
+    @staticmethod
+    def _estimate_size(value: Any) -> int:
+        """Rough byte-size estimate for analytics."""
+        try:
+            if isinstance(value, list):
+                return len(value) * 100
+            return len(json.dumps(value, default=str))
+        except Exception:
+            return 1000
+
+    # ------------------------------------------------------------------
+    # Core API (simple key-based)
+    # ------------------------------------------------------------------
+
+    def set(self, key: str, value: Any) -> None:
+        """Store *value* under a plain string *key*.
+
+        This is the simple interface expected by the test suite.
+        """
+        self._put_internal(key, value)
+
+    def get(self, key: str, **kwargs: Any) -> Optional[Any]:
+        """Retrieve value by plain string *key*.
+
+        Returns ``None`` on cache miss or expiry.
+        """
+        if kwargs:
+            key = self._hash_key(key, **kwargs)
+        return self._get_internal(key)
+
+    # ------------------------------------------------------------------
+    # Query-oriented API (hashed keys)
+    # ------------------------------------------------------------------
+
+    def put(self, query: str, value: Any, **kwargs: Any) -> None:
+        """Store *value* keyed by the hash of *query* + *kwargs*."""
+        hashed = self._hash_key(query, **kwargs)
+        self._put_internal(hashed, value)
+
+    def get_by_query(self, query: str, **kwargs: Any) -> Optional[Any]:
+        """Retrieve value keyed by the hash of *query* + *kwargs*."""
+        hashed = self._hash_key(query, **kwargs)
+        return self._get_internal(hashed)
+
+    # ------------------------------------------------------------------
+    # Internal get / put (shared logic)
+    # ------------------------------------------------------------------
+
+    def _get_internal(self, key: str) -> Optional[Any]:
+        if self._lock:
+            self._lock.acquire()
+        try:
+            if key not in self._cache:
+                self._misses += 1
+                return None
+
+            entry = self._cache[key]
+
+            if entry.is_expired(self.ttl_seconds):
+                del self._cache[key]
+                self._total_size_estimate -= entry.size_estimate
+                self._misses += 1
+                return None
+
+            self._cache.move_to_end(key)
+            entry.mark_accessed()
+            self._hits += 1
+            return entry.value
+        finally:
+            if self._lock:
+                self._lock.release()
+
+    def _put_internal(self, key: str, value: Any) -> None:
+        size_estimate = self._estimate_size(value)
+
+        if self._lock:
+            self._lock.acquire()
+        try:
+            if key in self._cache:
+                old = self._cache[key]
+                self._total_size_estimate -= old.size_estimate
+                del self._cache[key]
+
+            if len(self._cache) >= self.max_size:
+                _, evicted = self._cache.popitem(last=False)
+                self._total_size_estimate -= evicted.size_estimate
+                self._evictions += 1
+
+            self._cache[key] = CacheEntry(value, size_estimate)
+            self._total_size_estimate += size_estimate
+        finally:
+            if self._lock:
+                self._lock.release()
+
+    # ------------------------------------------------------------------
+    # Maintenance
+    # ------------------------------------------------------------------
+
+    def invalidate(self, query: str, **kwargs: Any) -> bool:
+        """Remove entry for *query*. Returns ``True`` if it existed."""
+        key = self._hash_key(query, **kwargs)
+        if self._lock:
+            self._lock.acquire()
+        try:
+            if key in self._cache:
+                entry = self._cache.pop(key)
+                self._total_size_estimate -= entry.size_estimate
+                return True
+            return False
+        finally:
+            if self._lock:
+                self._lock.release()
+
+    def clear(self) -> None:
+        """Remove all entries."""
+        if self._lock:
+            self._lock.acquire()
+        try:
+            self._cache.clear()
+            self._total_size_estimate = 0
+        finally:
+            if self._lock:
+                self._lock.release()
+
+    def evict_expired(self) -> int:
+        """Manually remove all expired entries. Returns count removed."""
+        if self.ttl_seconds is None:
+            return 0
+        if self._lock:
+            self._lock.acquire()
+        try:
+            expired = [
+                k for k, e in self._cache.items()
+                if e.is_expired(self.ttl_seconds)
+            ]
+            for k in expired:
+                entry = self._cache.pop(k)
+                self._total_size_estimate -= entry.size_estimate
+            return len(expired)
+        finally:
+            if self._lock:
+                self._lock.release()
+
+    # ------------------------------------------------------------------
+    # Stats
+    # ------------------------------------------------------------------
+
+    def get_stats(self) -> Dict[str, Any]:
+        """Return cache statistics snapshot."""
+        total = self._hits + self._misses
+        hit_rate = self._hits / total if total > 0 else 0.0
+
+        avg_access = 0.0
+        if self._cache:
+            avg_access = sum(e.access_count for e in self._cache.values()) / len(self._cache)
+
+        return {
+            "max_size": self.max_size,
+            "current_size": len(self._cache),
+            "ttl_seconds": self.ttl_seconds,
+            "hits": self._hits,
+            "misses": self._misses,
+            "hit_rate": hit_rate,
+            "evictions": self._evictions,
+            "total_size_estimate_kb": self._total_size_estimate / 1024,
+            "avg_access_count": avg_access,
+            "thread_safe": self.thread_safe,
+        }
+
+    def get_top_queries(self, limit: int = 10) -> List[Tuple[str, int]]:
+        """Return the *limit* most-accessed cache keys."""
+        if self._lock:
+            self._lock.acquire()
+        try:
+            items = [
+                (k, e.access_count) for k, e in self._cache.items()
+            ]
+            items.sort(key=lambda x: x[1], reverse=True)
+            return items[:limit]
+        finally:
+            if self._lock:
+                self._lock.release()

package/src/superlocalmemory/infra/event_bus.py

@@ -0,0 +1,381 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under the MIT License - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+"""EventBus -- Real-time event broadcasting for memory operations.
+
+Thread-safe singleton per DB path. In-memory deque buffer + SQLite persistence.
+Listener callbacks run on the emitter's thread.
+"""
+
+import json
+import logging
+import sqlite3
+import threading
+from collections import deque
+from datetime import datetime, timedelta
+from pathlib import Path
+from typing import Any, Callable, Dict, List, Optional
+
+logger = logging.getLogger("superlocalmemory.events")
+
+# Default retention windows (hours)
+DEFAULT_HOT_HOURS = 48
+DEFAULT_WARM_HOURS = 14 * 24  # 14 days
+DEFAULT_COLD_HOURS = 30 * 24  # 30 days
+
+# In-memory buffer size for real-time delivery
+EVENT_BUFFER_SIZE = 200
+
+# Valid event types (V3 superset)
+VALID_EVENT_TYPES = frozenset([
+    "memory.stored",        # New memory written (was memory.created in V2)
+    "memory.updated",       # Existing memory modified
+    "memory.deleted",       # Memory removed
+    "memory.recalled",      # Memory retrieved by an agent
+    "graph.updated",        # Knowledge graph rebuilt
+    "pattern.learned",      # New pattern detected
+    "agent.connected",      # New agent connects
+    "agent.disconnected",   # Agent disconnects
+    "trust.signal",         # V3: trust score change
+    "compliance.audit",     # V3: compliance event logged
+    "learning.feedback",    # V3: learning feedback received
+])
+
+
+class EventBus:
+    """
+    Central event bus for SuperLocalMemory V3.
+
+    Singleton per database path. Emits events to persistent storage and
+    in-memory listeners simultaneously. Thread-safe.
+    """
+
+    _instances: Dict[str, "EventBus"] = {}
+    _instances_lock = threading.Lock()
+
+    @classmethod
+    def get_instance(cls, db_path: Optional[Path] = None) -> "EventBus":
+        """Get or create the singleton EventBus for a database path."""
+        if db_path is None:
+            db_path = Path.home() / ".superlocalmemory" / "memory.db"
+
+        key = str(db_path)
+        with cls._instances_lock:
+            if key not in cls._instances:
+                cls._instances[key] = cls(db_path)
+            return cls._instances[key]
+
+    @classmethod
+    def reset_instance(cls, db_path: Optional[Path] = None) -> None:
+        """Remove and close a singleton instance. Used for testing."""
+        with cls._instances_lock:
+            if db_path is None:
+                cls._instances.clear()
+            else:
+                key = str(db_path)
+                if key in cls._instances:
+                    del cls._instances[key]
+
+    def __init__(self, db_path: Path) -> None:
+        """Initialize EventBus. Prefer get_instance() over direct construction."""
+        self.db_path = Path(db_path)
+        self._buffer: deque = deque(maxlen=EVENT_BUFFER_SIZE)
+        self._buffer_lock = threading.Lock()
+        self._event_counter = 0
+        self._counter_lock = threading.Lock()
+        self._listeners: List[Callable[[dict], None]] = []
+        self._listeners_lock = threading.Lock()
+        self._write_count = 0
+        self._last_prune = datetime.now()
+        self._init_schema()
+        logger.info("EventBus initialized: db=%s", self.db_path)
+
+    def _init_schema(self) -> None:
+        """Create the memory_events table if it does not exist."""
+        conn = sqlite3.connect(str(self.db_path))
+        try:
+            cur = conn.cursor()
+            cur.execute("""
+                CREATE TABLE IF NOT EXISTS memory_events (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT,
+                    event_type TEXT NOT NULL,
+                    memory_id INTEGER,
+                    source_agent TEXT DEFAULT 'user',
+                    source_protocol TEXT DEFAULT 'internal',
+                    payload TEXT,
+                    importance INTEGER DEFAULT 5,
+                    tier TEXT DEFAULT 'hot',
+                    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+                )
+            """)
+            cur.execute("CREATE INDEX IF NOT EXISTS idx_events_type ON memory_events(event_type)")
+            cur.execute("CREATE INDEX IF NOT EXISTS idx_events_created ON memory_events(created_at)")
+            cur.execute("CREATE INDEX IF NOT EXISTS idx_events_tier ON memory_events(tier)")
+            conn.commit()
+        finally:
+            conn.close()
+
+    def emit(
+        self,
+        event_type: str,
+        payload: Optional[Dict[str, Any]] = None,
+        memory_id: Optional[int] = None,
+        source_agent: str = "user",
+        source_protocol: str = "internal",
+        importance: int = 5,
+    ) -> Optional[int]:
+        """Emit an event to all subscribers and persist to database."""
+        if event_type not in VALID_EVENT_TYPES:
+            raise ValueError(
+                f"Invalid event type: {event_type}. "
+                f"Valid: {', '.join(sorted(VALID_EVENT_TYPES))}"
+            )
+
+        importance = max(1, min(10, importance))
+
+        now = datetime.now().isoformat()
+        with self._counter_lock:
+            self._event_counter += 1
+            seq = self._event_counter
+
+        event: Dict[str, Any] = {
+            "seq": seq,
+            "event_type": event_type,
+            "memory_id": memory_id,
+            "source_agent": source_agent,
+            "source_protocol": source_protocol,
+            "payload": payload or {},
+            "importance": importance,
+            "timestamp": now,
+        }
+
+        # 1. Persist
+        event_id = self._persist_event(event)
+        if event_id is not None:
+            event["id"] = event_id
+
+        # 2. Buffer
+        with self._buffer_lock:
+            self._buffer.append(event)
+
+        # 3. Notify
+        self._notify_listeners(event)
+
+        logger.debug(
+            "Event emitted: type=%s id=%s memory_id=%s",
+            event_type, event_id, memory_id,
+        )
+
+        # Auto-prune heuristic
+        self._write_count += 1
+        if (
+            self._write_count >= 100
+            or (datetime.now() - self._last_prune).total_seconds() > 86400
+        ):
+            try:
+                self.prune_events()
+            except Exception:
+                pass
+            self._write_count = 0
+            self._last_prune = datetime.now()
+
+        return event_id
+
+    # Alias for V3 compatibility
+    publish = emit
+
+    def _persist_event(self, event: dict) -> Optional[int]:
+        """Persist event to the memory_events table. Returns row id or None."""
+        try:
+            conn = sqlite3.connect(str(self.db_path))
+            try:
+                cur = conn.cursor()
+                cur.execute(
+                    "INSERT INTO memory_events (event_type, memory_id, source_agent,"
+                    " source_protocol, payload, importance, tier, created_at)"
+                    " VALUES (?, ?, ?, ?, ?, ?, 'hot', ?)",
+                    (event["event_type"], event.get("memory_id"),
+                     event["source_agent"], event["source_protocol"],
+                     json.dumps(event["payload"]), event["importance"],
+                     event["timestamp"]),
+                )
+                conn.commit()
+                return cur.lastrowid
+            finally:
+                conn.close()
+        except Exception as exc:
+            logger.error("Failed to persist event: %s", exc)
+            return None
+
+    def add_listener(self, callback: Callable[[dict], None]) -> None:
+        """Register a listener that receives every emitted event."""
+        with self._listeners_lock:
+            self._listeners.append(callback)
+
+    # Alias
+    subscribe = add_listener
+
+    def remove_listener(self, callback: Callable[[dict], None]) -> None:
+        """Remove a previously registered listener."""
+        with self._listeners_lock:
+            try:
+                self._listeners.remove(callback)
+            except ValueError:
+                pass
+
+    # Alias
+    unsubscribe = remove_listener
+
+    def _notify_listeners(self, event: dict) -> None:
+        """Call all registered listeners. Errors are logged, not raised."""
+        with self._listeners_lock:
+            listeners = list(self._listeners)
+
+        for listener in listeners:
+            try:
+                listener(event)
+            except Exception as exc:
+                logger.error("Event listener failed: %s", exc)
+
+    def get_recent_events(
+        self,
+        since_id: Optional[int] = None,
+        limit: int = 50,
+        event_type: Optional[str] = None,
+    ) -> List[dict]:
+        """Get recent events from the database."""
+        limit = min(limit, 200)
+
+        try:
+            conn = sqlite3.connect(str(self.db_path))
+            try:
+                cur = conn.cursor()
+
+                query = ("SELECT id, event_type, memory_id, source_agent,"
+                         " source_protocol, payload, importance, tier, created_at"
+                         " FROM memory_events WHERE 1=1")
+                params: List[Any] = []
+
+                if since_id is not None:
+                    query += " AND id > ?"
+                    params.append(since_id)
+
+                if event_type:
+                    query += " AND event_type = ?"
+                    params.append(event_type)
+
+                query += " ORDER BY id ASC LIMIT ?"
+                params.append(limit)
+
+                cur.execute(query, params)
+                rows = cur.fetchall()
+            finally:
+                conn.close()
+
+            events: List[dict] = []
+            for row in rows:
+                try:
+                    parsed = json.loads(row[5]) if row[5] else {}
+                except (json.JSONDecodeError, TypeError):
+                    parsed = {}
+                events.append({
+                    "id": row[0], "event_type": row[1], "memory_id": row[2],
+                    "source_agent": row[3], "source_protocol": row[4],
+                    "payload": parsed, "importance": row[6],
+                    "tier": row[7], "timestamp": row[8],
+                })
+            return events
+
+        except Exception as exc:
+            logger.error("Failed to get recent events: %s", exc)
+            return []
+
+    def get_buffered_events(self, since_seq: int = 0) -> List[dict]:
+        """Get events from the in-memory buffer (no DB hit)."""
+        with self._buffer_lock:
+            return [e for e in self._buffer if e.get("seq", 0) > since_seq]
+
+    def get_event_stats(self) -> dict:
+        """Get event system statistics."""
+        try:
+            conn = sqlite3.connect(str(self.db_path))
+            try:
+                cur = conn.cursor()
+
+                total = cur.execute("SELECT COUNT(*) FROM memory_events").fetchone()[0]
+                cur.execute("SELECT event_type, COUNT(*) FROM memory_events GROUP BY event_type")
+                by_type = dict(cur.fetchall())
+                cur.execute("SELECT tier, COUNT(*) FROM memory_events GROUP BY tier")
+                by_tier = dict(cur.fetchall())
+                cur.execute("SELECT COUNT(*) FROM memory_events WHERE created_at >= datetime('now', '-24 hours')")
+                last_24h = cur.fetchone()[0]
+            finally:
+                conn.close()
+
+            return {
+                **by_type,
+                "total_events": total,
+                "events_last_24h": last_24h,
+                "by_type": by_type,
+                "by_tier": by_tier,
+                "buffer_size": len(self._buffer),
+                "listener_count": len(self._listeners),
+            }
+
+        except Exception as exc:
+            logger.error("Failed to get event stats: %s", exc)
+            return {"total_events": 0, "error": str(exc)}
+
+    def prune_events(
+        self,
+        hot_hours: int = DEFAULT_HOT_HOURS,
+        warm_hours: int = DEFAULT_WARM_HOURS,
+        cold_hours: int = DEFAULT_COLD_HOURS,
+    ) -> dict:
+        """Apply tiered retention policy to persisted events."""
+        try:
+            conn = sqlite3.connect(str(self.db_path))
+            try:
+                cur = conn.cursor()
+                now = datetime.now()
+                stats = {"hot_to_warm": 0, "warm_to_cold": 0, "archived": 0}
+
+                # Hot -> Warm: older than hot_hours, importance < 5
+                warm_cutoff = (now - timedelta(hours=hot_hours)).isoformat()
+                cur.execute(
+                    "UPDATE memory_events SET tier = 'warm' "
+                    "WHERE tier = 'hot' AND created_at < ? AND importance < 5",
+                    (warm_cutoff,),
+                )
+                stats["hot_to_warm"] = cur.rowcount
+
+                # Warm -> Cold: delete warm events older than warm_hours
+                cold_cutoff = (now - timedelta(hours=warm_hours)).isoformat()
+                cur.execute(
+                    "DELETE FROM memory_events "
+                    "WHERE tier = 'warm' AND created_at < ?",
+                    (cold_cutoff,),
+                )
+                stats["warm_to_cold"] = cur.rowcount
+
+                # Archive: delete everything older than cold_hours
+                archive_cutoff = (now - timedelta(hours=cold_hours)).isoformat()
+                cur.execute(
+                    "DELETE FROM memory_events WHERE created_at < ?",
+                    (archive_cutoff,),
+                )
+                stats["archived"] = cur.rowcount
+
+                conn.commit()
+            finally:
+                conn.close()
+
+            logger.info(
+                "Prune complete: hot->warm=%d warm->cold=%d archived=%d",
+                stats["hot_to_warm"], stats["warm_to_cold"], stats["archived"],
+            )
+            return stats
+
+        except Exception as exc:
+            logger.error("Event pruning failed: %s", exc)
+            return {"error": str(exc)}