superlocalmemory 4.1.13 → 4.1.14

package/src/superlocalmemory/infra/backup_obligations.py

@@ -1,423 +0,0 @@
-# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
-# Licensed under AGPL-3.0-or-later - see LICENSE file
-# Part of SuperLocalMemory V4 | https://qualixar.com
-
-"""Backup erasure obligation ledger — GDPR Art.17 backup residue tracking.
-
-After Art.17 erasure the live stores are clean, but rotating backup snapshots
-still hold the erased personal data.  This module tracks ``outstanding
-obligations`` — one per (profile_id, snapshot_path) pair — so that:
-
-  1. The erasure receipt accurately declares completeness as FALSE while any
-     snapshot still contains the data (C1 gap closure).
-
-  2. A restore that would resurrect erased data is intercepted and the profile
-     data re-erased from the restored store before any caller can read it
-     (restore-replay invariant).
-
-  3. Obligations age out automatically when a snapshot exceeds the configured
-     retention window, keeping the ledger finite.
-
-IMPORTANT: backup_obligations.db is intentionally NOT in MANAGED_DATABASES
-and is therefore never backed up itself.  This prevents a restore from loading
-stale obligation state.
-"""
-
-from __future__ import annotations
-
-import logging
-import sqlite3
-import time
-import uuid
-from pathlib import Path
-
-logger = logging.getLogger("superlocalmemory.infra.backup_obligations")
-
-_OBLIGATION_DB_NAME = "backup_obligations.db"
-
-_DDL = """
-CREATE TABLE IF NOT EXISTS backup_obligations (
-    obligation_id    TEXT PRIMARY KEY,
-    profile_id       TEXT NOT NULL,
-    erasure_id       TEXT NOT NULL,
-    snapshot_path    TEXT NOT NULL,
-    snapshot_epoch   INTEGER NOT NULL,
-    retention_days   INTEGER NOT NULL DEFAULT 90,
-    recorded_at      REAL NOT NULL,
-    status           TEXT NOT NULL DEFAULT 'pending',
-    discharged_at    REAL,
-    discharge_reason TEXT
-);
-CREATE INDEX IF NOT EXISTS idx_bko_profile
-    ON backup_obligations(profile_id, status);
-CREATE INDEX IF NOT EXISTS idx_bko_snapshot
-    ON backup_obligations(snapshot_path, status);
-CREATE INDEX IF NOT EXISTS idx_bko_erasure
-    ON backup_obligations(erasure_id);
-"""
-
-
-class BackupObligationStore:
-    """Persistent ledger of outstanding erasure obligations against snapshots.
-
-    An obligation is created for each backup snapshot that was found to contain
-    data for a profile that has since been Art.17-erased from the live stores.
-    The obligation is discharged when:
-      * The snapshot ages past the configured retention window (auto-discharge).
-      * The snapshot is restored and the profile data is re-erased during
-        restore-replay (explicit discharge).
-
-    The store lives at ``<data_root>/backup_obligations.db``.  It is a plain
-    SQLite file that is NOT in MANAGED_DATABASES and therefore never appears
-    inside a backup set.
-    """
-
-    def __init__(self, data_root: Path) -> None:
-        self._db_path = Path(data_root) / _OBLIGATION_DB_NAME
-        self._ensure_schema()
-
-    # ------------------------------------------------------------------
-    # Schema
-    # ------------------------------------------------------------------
-
-    def _connect(self) -> sqlite3.Connection:
-        conn = sqlite3.connect(str(self._db_path))
-        conn.row_factory = sqlite3.Row
-        conn.execute("PRAGMA journal_mode=WAL")
-        conn.execute("PRAGMA foreign_keys=ON")
-        conn.execute("PRAGMA synchronous=NORMAL")
-        return conn
-
-    def _ensure_schema(self) -> None:
-        try:
-            with self._connect() as conn:
-                for stmt in _DDL.strip().split(";"):
-                    stmt = stmt.strip()
-                    if stmt:
-                        conn.execute(stmt)
-                conn.commit()
-        except Exception as exc:  # noqa: BLE001
-            logger.warning("BackupObligationStore: schema init failed: %s", exc)
-
-    # ------------------------------------------------------------------
-    # Write path
-    # ------------------------------------------------------------------
-
-    def record(
-        self,
-        profile_id: str,
-        erasure_id: str,
-        snapshot_path: str,
-        snapshot_epoch: int,
-        retention_days: int = 90,
-    ) -> str:
-        """Record one outstanding obligation.  Returns the obligation_id."""
-        oid = uuid.uuid4().hex
-        try:
-            with self._connect() as conn:
-                conn.execute(
-                    "INSERT OR IGNORE INTO backup_obligations "
-                    "(obligation_id, profile_id, erasure_id, snapshot_path, "
-                    " snapshot_epoch, retention_days, recorded_at, status) "
-                    "VALUES (?, ?, ?, ?, ?, ?, ?, 'pending')",
-                    (
-                        oid, profile_id, erasure_id, str(snapshot_path),
-                        int(snapshot_epoch), int(retention_days), time.time(),
-                    ),
-                )
-                conn.commit()
-        except Exception as exc:  # noqa: BLE001
-            logger.warning("BackupObligationStore.record failed: %s", exc)
-        return oid
-
-    def discharge(self, obligation_id: str, reason: str) -> None:
-        """Mark a single obligation as discharged."""
-        try:
-            with self._connect() as conn:
-                conn.execute(
-                    "UPDATE backup_obligations "
-                    "SET status='discharged', discharged_at=?, discharge_reason=? "
-                    "WHERE obligation_id=?",
-                    (time.time(), reason, obligation_id),
-                )
-                conn.commit()
-        except Exception as exc:  # noqa: BLE001
-            logger.warning("BackupObligationStore.discharge failed: %s", exc)
-
-    def discharge_for_snapshot(self, snapshot_path: str, reason: str) -> int:
-        """Discharge ALL pending obligations for a snapshot path.  Returns count."""
-        try:
-            with self._connect() as conn:
-                cur = conn.execute(
-                    "UPDATE backup_obligations "
-                    "SET status='discharged', discharged_at=?, discharge_reason=? "
-                    "WHERE snapshot_path=? AND status='pending'",
-                    (time.time(), reason, str(snapshot_path)),
-                )
-                conn.commit()
-                return cur.rowcount
-        except Exception as exc:  # noqa: BLE001
-            logger.warning("BackupObligationStore.discharge_for_snapshot: %s", exc)
-            return 0
-
-    # ------------------------------------------------------------------
-    # Read path
-    # ------------------------------------------------------------------
-
-    def _discharge_aged_out(self) -> int:
-        """Auto-discharge obligations whose snapshot has aged past retention."""
-        try:
-            with self._connect() as conn:
-                now = time.time()
-                cur = conn.execute(
-                    "UPDATE backup_obligations "
-                    "SET status='discharged', discharged_at=?, "
-                    "    discharge_reason='snapshot_aged_out' "
-                    "WHERE status='pending' "
-                    "  AND (snapshot_epoch + retention_days * 86400) < ?",
-                    (now, now),
-                )
-                conn.commit()
-                return cur.rowcount
-        except Exception as exc:  # noqa: BLE001
-            logger.warning("BackupObligationStore._discharge_aged_out: %s", exc)
-            return 0
-
-    def count_pending(self, profile_id: str) -> int:
-        """Count pending obligations for *profile_id* (auto-discharges aged-out first).
-
-        Fail-closed: if the store is unreadable, returns 1 to block completeness.
-        """
-        self._discharge_aged_out()
-        try:
-            with self._connect() as conn:
-                row = conn.execute(
-                    "SELECT COUNT(*) FROM backup_obligations "
-                    "WHERE profile_id=? AND status='pending'",
-                    (profile_id,),
-                ).fetchone()
-                return int(row[0]) if row else 0
-        except Exception as exc:  # noqa: BLE001
-            logger.warning(
-                "BackupObligationStore.count_pending failed: %s — "
-                "returning 1 to block completeness claim",
-                exc,
-            )
-            return 1  # fail-closed
-
-    def list_pending_for_snapshot(self, snapshot_path: str) -> list[dict]:
-        """Return all pending obligations for *snapshot_path*."""
-        try:
-            with self._connect() as conn:
-                rows = conn.execute(
-                    "SELECT * FROM backup_obligations "
-                    "WHERE snapshot_path=? AND status='pending'",
-                    (str(snapshot_path),),
-                ).fetchall()
-                return [dict(r) for r in rows]
-        except Exception as exc:  # noqa: BLE001
-            logger.warning("BackupObligationStore.list_pending_for_snapshot: %s", exc)
-            return []
-
-    def list_pending_for_profile(self, profile_id: str) -> list[dict]:
-        """Return all pending obligations for *profile_id* (auto-discharges first)."""
-        self._discharge_aged_out()
-        try:
-            with self._connect() as conn:
-                rows = conn.execute(
-                    "SELECT * FROM backup_obligations "
-                    "WHERE profile_id=? AND status='pending'",
-                    (profile_id,),
-                ).fetchall()
-                return [dict(r) for r in rows]
-        except Exception as exc:  # noqa: BLE001
-            logger.warning("BackupObligationStore.list_pending_for_profile: %s", exc)
-            return []
-
-
-# ---------------------------------------------------------------------------
-# Snapshot helpers (used by gdpr.py and backup.py)
-# ---------------------------------------------------------------------------
-
-
-def scan_backup_snapshots_for_profile(
-    backup_dir: Path,
-    profile_id: str,
-) -> list[tuple[str, int]]:
-    """Scan *backup_dir* for snapshots containing *profile_id*'s data.
-
-    Returns a list of (snapshot_path_str, epoch) tuples.  Handles both legacy
-    per-file backups (``*.db`` files) and new-style BackupCoordinator sets
-    (``backup_XXXX/`` directories containing a ``manifest.json``).
-    """
-    backup_dir = Path(backup_dir)
-    results: list[tuple[str, int]] = []
-
-    if not backup_dir.exists():
-        return results
-
-    # New-style backup sets: backup_XXXX/ directories with manifest.json.
-    # Record obligation at the set-directory level (one obligation per set).
-    for candidate in sorted(backup_dir.iterdir()):
-        if not candidate.is_dir() or not (candidate / "manifest.json").exists():
-            continue
-        hit = False
-        for db_file in candidate.glob("*.db"):
-            if _snapshot_db_contains_profile(db_file, profile_id):
-                hit = True
-                break
-        if hit:
-            epoch = int(candidate.stat().st_mtime)
-            results.append((str(candidate), epoch))
-
-    # Legacy per-file backups: ``memory-YYYYMMDD-HHMMSS.db``, etc.
-    for db_file in sorted(backup_dir.glob("*.db")):
-        if db_file.name.startswith("."):
-            continue
-        if _snapshot_db_contains_profile(db_file, profile_id):
-            epoch = int(db_file.stat().st_mtime)
-            results.append((str(db_file), epoch))
-
-    return results
-
-
-def _snapshot_db_contains_profile(db_path: Path, profile_id: str) -> bool:
-    """Return True if the SQLite file at *db_path* contains rows for *profile_id*.
-
-    Opens read-only (immutable) to avoid side-effects.  Returns False on any
-    error so an unreadable snapshot does not block erasure; caller logs the
-    warning separately.
-    """
-    try:
-        uri = f"file:{db_path}?mode=ro"
-        conn = sqlite3.connect(uri, uri=True, timeout=5)
-        try:
-            tables = {
-                row[0]
-                for row in conn.execute(
-                    "SELECT name FROM sqlite_master WHERE type='table'"
-                ).fetchall()
-            }
-            for table in ("profiles", "atomic_facts", "memories", "graph_nodes"):
-                if table not in tables:
-                    continue
-                cols = {
-                    row[1]
-                    for row in conn.execute(f"PRAGMA table_info({table})").fetchall()
-                }
-                if "profile_id" not in cols:
-                    # graph_nodes has no profile_id — flag unconditionally as personal
-                    if table == "graph_nodes":
-                        row = conn.execute(
-                            "SELECT 1 FROM graph_nodes LIMIT 1"
-                        ).fetchone()
-                        if row is not None:
-                            return True
-                    continue
-                row = conn.execute(
-                    f"SELECT 1 FROM {table} WHERE profile_id=? LIMIT 1",
-                    (profile_id,),
-                ).fetchone()
-                if row is not None:
-                    return True
-        finally:
-            conn.close()
-    except Exception as exc:  # noqa: BLE001
-        logger.warning(
-            "_snapshot_db_contains_profile: error reading %s: %s — treating as clean",
-            db_path.name, exc,
-        )
-    return False
-
-
-def erase_profile_from_snapshot(db_path: Path, profile_id: str) -> dict[str, int]:
-    """Delete all *profile_id* rows from a snapshot SQLite file.
-
-    Used during restore-replay.  Raises on any fatal error so the caller can
-    refuse to mark the restore complete when personal data cannot be purged.
-    """
-    deleted: dict[str, int] = {}
-    # isolation_level=None → autocommit so VACUUM can run outside any transaction.
-    conn = sqlite3.connect(str(db_path))
-    conn.isolation_level = None
-    try:
-        conn.execute("PRAGMA foreign_keys=OFF")
-        tables = [
-            row[0]
-            for row in conn.execute(
-                "SELECT name FROM sqlite_master WHERE type='table'"
-            ).fetchall()
-            if not row[0].startswith("sqlite_")
-        ]
-        conn.execute("BEGIN")
-        for table in tables:
-            cols = {
-                row[1]
-                for row in conn.execute(f"PRAGMA table_info({table})").fetchall()
-            }
-            if "profile_id" not in cols:
-                continue
-            cur = conn.execute(
-                f"DELETE FROM {table} WHERE profile_id=?", (profile_id,)
-            )
-            if cur.rowcount:
-                deleted[table] = cur.rowcount
-        conn.execute("PRAGMA foreign_keys=ON")
-        conn.execute("COMMIT")
-        # VACUUM must run in autocommit (no active transaction).
-        conn.execute("VACUUM")
-    except Exception:
-        try:
-            conn.execute("ROLLBACK")
-        except Exception:  # noqa: BLE001
-            pass
-        conn.close()
-        raise
-    else:
-        conn.close()
-    return deleted
-
-
-def erase_code_graph_from_snapshot(db_path: Path) -> dict[str, int]:
-    """Wipe all tables from a code_graph.db snapshot (no profile_id column).
-
-    The code graph is installation-level personal data (repo paths, symbol
-    names).  On erasure or restore-replay the entire graph is cleared.
-    """
-    deleted: dict[str, int] = {}
-    if not db_path.exists():
-        return deleted
-    conn = sqlite3.connect(str(db_path))
-    conn.isolation_level = None  # autocommit so VACUUM runs outside any transaction
-    try:
-        conn.execute("PRAGMA foreign_keys=OFF")
-        tables = [
-            row[0]
-            for row in conn.execute(
-                "SELECT name FROM sqlite_master WHERE type='table' "
-                "AND name NOT LIKE 'sqlite_%'"
-            ).fetchall()
-        ]
-        conn.execute("BEGIN")
-        for table in tables:
-            # Skip FTS virtual tables — deleting the base table handles them
-            if table.endswith(("_fts", "_fts_data", "_fts_idx", "_fts_content",
-                                "_fts_docsize", "_fts_config")):
-                continue
-            cur = conn.execute(f"DELETE FROM {table}")
-            if cur.rowcount:
-                deleted[table] = cur.rowcount
-        conn.execute("PRAGMA foreign_keys=ON")
-        conn.execute("COMMIT")
-        # VACUUM must run in autocommit (no active transaction).
-        conn.execute("VACUUM")
-    except Exception:
-        try:
-            conn.execute("ROLLBACK")
-        except Exception:  # noqa: BLE001
-            pass
-        conn.close()
-        raise
-    else:
-        conn.close()
-    return deleted

package/src/superlocalmemory/infra/cache_manager.py

@@ -1,267 +0,0 @@
-# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
-# Licensed under AGPL-3.0-or-later - see LICENSE file
-# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
-"""LRU cache with optional TTL for search-result caching.
-
-Key features:
-    * O(1) get / set via ``OrderedDict``
-    * Time-to-live expiry per entry
-    * Size-based eviction (LRU)
-    * Optional thread safety
-    * Hit / miss / eviction statistics
-"""
-
-import hashlib
-import json
-import logging
-import time
-from collections import OrderedDict
-from threading import RLock
-from typing import Any, Dict, List, Optional, Tuple
-
-logger = logging.getLogger("superlocalmemory.cache")
-
-
-class CacheEntry:
-    """Single cache entry with metadata."""
-
-    __slots__ = ["value", "timestamp", "access_count", "size_estimate"]
-
-    def __init__(self, value: Any, size_estimate: int = 0) -> None:
-        self.value = value
-        self.timestamp = time.time()
-        self.access_count = 0
-        self.size_estimate = size_estimate
-
-    def is_expired(self, ttl_seconds: Optional[float]) -> bool:
-        """Return ``True`` when the entry has exceeded *ttl_seconds*."""
-        if ttl_seconds is None:
-            return False
-        return (time.time() - self.timestamp) > ttl_seconds
-
-    def mark_accessed(self) -> None:
-        """Increment access counter."""
-        self.access_count += 1
-
-
-class CacheManager:
-    """LRU cache manager with TTL support.
-
-    Args:
-        max_size: Maximum number of entries before LRU eviction.
-        ttl_seconds: Per-entry time-to-live (``None`` = never expire).
-        thread_safe: Wrap operations in a reentrant lock.
-    """
-
-    def __init__(
-        self,
-        max_size: int = 100,
-        ttl_seconds: Optional[float] = 300,
-        thread_safe: bool = False,
-    ) -> None:
-        self.max_size = max(1, max_size)
-        self.ttl_seconds = ttl_seconds
-        self.thread_safe = thread_safe
-
-        self._cache: OrderedDict[str, CacheEntry] = OrderedDict()
-        self._lock: Optional[RLock] = RLock() if thread_safe else None
-
-        # Statistics
-        self._hits = 0
-        self._misses = 0
-        self._evictions = 0
-        self._total_size_estimate = 0
-
-    # ------------------------------------------------------------------
-    # Key helpers
-    # ------------------------------------------------------------------
-
-    @staticmethod
-    def _hash_key(query: str, **kwargs: Any) -> str:
-        """Deterministic cache key from *query* + keyword args."""
-        key_data = {"query": query, **kwargs}
-        key_str = json.dumps(key_data, sort_keys=True)
-        return hashlib.sha256(key_str.encode()).hexdigest()[:16]
-
-    @staticmethod
-    def _estimate_size(value: Any) -> int:
-        """Rough byte-size estimate for analytics."""
-        try:
-            if isinstance(value, list):
-                return len(value) * 100
-            return len(json.dumps(value, default=str))
-        except Exception:
-            return 1000
-
-    # ------------------------------------------------------------------
-    # Core API (simple key-based)
-    # ------------------------------------------------------------------
-
-    def set(self, key: str, value: Any) -> None:
-        """Store *value* under a plain string *key*.
-
-        This is the simple interface expected by the test suite.
-        """
-        self._put_internal(key, value)
-
-    def get(self, key: str, **kwargs: Any) -> Optional[Any]:
-        """Retrieve value by plain string *key*.
-
-        Returns ``None`` on cache miss or expiry.
-        """
-        if kwargs:
-            key = self._hash_key(key, **kwargs)
-        return self._get_internal(key)
-
-    # ------------------------------------------------------------------
-    # Query-oriented API (hashed keys)
-    # ------------------------------------------------------------------
-
-    def put(self, query: str, value: Any, **kwargs: Any) -> None:
-        """Store *value* keyed by the hash of *query* + *kwargs*."""
-        hashed = self._hash_key(query, **kwargs)
-        self._put_internal(hashed, value)
-
-    def get_by_query(self, query: str, **kwargs: Any) -> Optional[Any]:
-        """Retrieve value keyed by the hash of *query* + *kwargs*."""
-        hashed = self._hash_key(query, **kwargs)
-        return self._get_internal(hashed)
-
-    # ------------------------------------------------------------------
-    # Internal get / put (shared logic)
-    # ------------------------------------------------------------------
-
-    def _get_internal(self, key: str) -> Optional[Any]:
-        if self._lock:
-            self._lock.acquire()
-        try:
-            if key not in self._cache:
-                self._misses += 1
-                return None
-
-            entry = self._cache[key]
-
-            if entry.is_expired(self.ttl_seconds):
-                del self._cache[key]
-                self._total_size_estimate -= entry.size_estimate
-                self._misses += 1
-                return None
-
-            self._cache.move_to_end(key)
-            entry.mark_accessed()
-            self._hits += 1
-            return entry.value
-        finally:
-            if self._lock:
-                self._lock.release()
-
-    def _put_internal(self, key: str, value: Any) -> None:
-        size_estimate = self._estimate_size(value)
-
-        if self._lock:
-            self._lock.acquire()
-        try:
-            if key in self._cache:
-                old = self._cache[key]
-                self._total_size_estimate -= old.size_estimate
-                del self._cache[key]
-
-            if len(self._cache) >= self.max_size:
-                _, evicted = self._cache.popitem(last=False)
-                self._total_size_estimate -= evicted.size_estimate
-                self._evictions += 1
-
-            self._cache[key] = CacheEntry(value, size_estimate)
-            self._total_size_estimate += size_estimate
-        finally:
-            if self._lock:
-                self._lock.release()
-
-    # ------------------------------------------------------------------
-    # Maintenance
-    # ------------------------------------------------------------------
-
-    def invalidate(self, query: str, **kwargs: Any) -> bool:
-        """Remove entry for *query*. Returns ``True`` if it existed."""
-        key = self._hash_key(query, **kwargs)
-        if self._lock:
-            self._lock.acquire()
-        try:
-            if key in self._cache:
-                entry = self._cache.pop(key)
-                self._total_size_estimate -= entry.size_estimate
-                return True
-            return False
-        finally:
-            if self._lock:
-                self._lock.release()
-
-    def clear(self) -> None:
-        """Remove all entries."""
-        if self._lock:
-            self._lock.acquire()
-        try:
-            self._cache.clear()
-            self._total_size_estimate = 0
-        finally:
-            if self._lock:
-                self._lock.release()
-
-    def evict_expired(self) -> int:
-        """Manually remove all expired entries. Returns count removed."""
-        if self.ttl_seconds is None:
-            return 0
-        if self._lock:
-            self._lock.acquire()
-        try:
-            expired = [
-                k for k, e in self._cache.items()
-                if e.is_expired(self.ttl_seconds)
-            ]
-            for k in expired:
-                entry = self._cache.pop(k)
-                self._total_size_estimate -= entry.size_estimate
-            return len(expired)
-        finally:
-            if self._lock:
-                self._lock.release()
-
-    # ------------------------------------------------------------------
-    # Stats
-    # ------------------------------------------------------------------
-
-    def get_stats(self) -> Dict[str, Any]:
-        """Return cache statistics snapshot."""
-        total = self._hits + self._misses
-        hit_rate = self._hits / total if total > 0 else 0.0
-
-        avg_access = 0.0
-        if self._cache:
-            avg_access = sum(e.access_count for e in self._cache.values()) / len(self._cache)
-
-        return {
-            "max_size": self.max_size,
-            "current_size": len(self._cache),
-            "ttl_seconds": self.ttl_seconds,
-            "hits": self._hits,
-            "misses": self._misses,
-            "hit_rate": hit_rate,
-            "evictions": self._evictions,
-            "total_size_estimate_kb": self._total_size_estimate / 1024,
-            "avg_access_count": avg_access,
-            "thread_safe": self.thread_safe,
-        }
-
-    def get_top_queries(self, limit: int = 10) -> List[Tuple[str, int]]:
-        """Return the *limit* most-accessed cache keys."""
-        if self._lock:
-            self._lock.acquire()
-        try:
-            items = [
-                (k, e.access_count) for k, e in self._cache.items()
-            ]
-            items.sort(key=lambda x: x[1], reverse=True)
-            return items[:limit]
-        finally:
-            if self._lock:
-                self._lock.release()