superlocalmemory 2.7.6 → 2.8.0

package/src/compliance/audit_db.py

@@ -0,0 +1,215 @@
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
+"""Audit database management with hash chain tamper detection.
+
+Provides a tamper-evident audit trail stored in a separate audit.db.
+Each entry's hash incorporates the previous entry's hash, forming a
+chain that can detect any modification to historical records.
+"""
+import hashlib
+import json
+import sqlite3
+import threading
+from datetime import datetime, timezone
+from typing import Any, Dict, List, Optional
+
+
+_GENESIS = "genesis"
+
+_SCHEMA = """
+CREATE TABLE IF NOT EXISTS audit_events (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    event_type TEXT NOT NULL,
+    actor TEXT NOT NULL,
+    resource_id INTEGER,
+    details TEXT DEFAULT '{}',
+    prev_hash TEXT NOT NULL,
+    entry_hash TEXT NOT NULL,
+    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+)
+"""
+
+
+def _compute_hash(
+    event_type: str,
+    actor: str,
+    resource_id: Optional[int],
+    details: str,
+    prev_hash: str,
+    created_at: str,
+) -> str:
+    """Compute SHA-256 hash for an audit entry."""
+    payload = (
+        f"{event_type}{actor}{resource_id}{details}{prev_hash}{created_at}"
+    )
+    return hashlib.sha256(payload.encode("utf-8")).hexdigest()
+
+
+class AuditDB:
+    """Manages audit.db -- tamper-evident compliance audit trail.
+
+    The hash chain guarantees that any modification to a stored event
+    will be detected by verify_chain(). The first entry uses a fixed
+    genesis value as its previous hash.
+    """
+
+    def __init__(self, db_path: Optional[str] = None):
+        self._db_path = db_path
+        self._lock = threading.Lock()
+        if db_path:
+            self._init_db()
+
+    # ------------------------------------------------------------------
+    # Internal helpers
+    # ------------------------------------------------------------------
+
+    def _get_conn(self) -> sqlite3.Connection:
+        conn = sqlite3.connect(self._db_path)
+        conn.execute("PRAGMA journal_mode=WAL")
+        conn.execute("PRAGMA foreign_keys=ON")
+        conn.row_factory = sqlite3.Row
+        return conn
+
+    def _init_db(self) -> None:
+        conn = self._get_conn()
+        try:
+            conn.executescript(_SCHEMA)
+            conn.commit()
+        finally:
+            conn.close()
+
+    # ------------------------------------------------------------------
+    # Public API
+    # ------------------------------------------------------------------
+
+    def log_event(
+        self,
+        event_type: str,
+        actor: str = "system",
+        resource_id: Optional[int] = None,
+        details: Optional[Dict[str, Any]] = None,
+    ) -> int:
+        """Append an event to the audit trail and return its row id."""
+        details_str = json.dumps(details or {}, sort_keys=True)
+        created_at = datetime.now(timezone.utc).isoformat()
+
+        with self._lock:
+            conn = self._get_conn()
+            try:
+                # Fetch the hash of the most recent entry (or genesis)
+                row = conn.execute(
+                    "SELECT entry_hash FROM audit_events "
+                    "ORDER BY id DESC LIMIT 1"
+                ).fetchone()
+                prev_hash = row["entry_hash"] if row else _GENESIS
+
+                entry_hash = _compute_hash(
+                    event_type, actor, resource_id,
+                    details_str, prev_hash, created_at,
+                )
+
+                cursor = conn.execute(
+                    "INSERT INTO audit_events "
+                    "(event_type, actor, resource_id, details, "
+                    " prev_hash, entry_hash, created_at) "
+                    "VALUES (?, ?, ?, ?, ?, ?, ?)",
+                    (
+                        event_type, actor, resource_id,
+                        details_str, prev_hash, entry_hash, created_at,
+                    ),
+                )
+                conn.commit()
+                return cursor.lastrowid
+            finally:
+                conn.close()
+
+    def query_events(
+        self,
+        event_type: Optional[str] = None,
+        actor: Optional[str] = None,
+        resource_id: Optional[int] = None,
+        limit: int = 100,
+    ) -> List[Dict[str, Any]]:
+        """Query audit events with optional filters."""
+        clauses: List[str] = []
+        params: List[Any] = []
+
+        if event_type is not None:
+            clauses.append("event_type = ?")
+            params.append(event_type)
+        if actor is not None:
+            clauses.append("actor = ?")
+            params.append(actor)
+        if resource_id is not None:
+            clauses.append("resource_id = ?")
+            params.append(resource_id)
+
+        where = f"WHERE {' AND '.join(clauses)}" if clauses else ""
+        sql = (
+            f"SELECT id, event_type, actor, resource_id, details, "
+            f"prev_hash, entry_hash, created_at "
+            f"FROM audit_events {where} "
+            f"ORDER BY id DESC LIMIT ?"
+        )
+        params.append(limit)
+
+        conn = self._get_conn()
+        try:
+            rows = conn.execute(sql, params).fetchall()
+            return [dict(r) for r in rows]
+        finally:
+            conn.close()
+
+    def verify_chain(self) -> Dict[str, Any]:
+        """Verify the integrity of the entire hash chain.
+
+        Returns a dict with:
+            valid            -- bool, True if chain is intact
+            entries_checked  -- int, number of entries verified
+            error            -- str or None, description of first failure
+        """
+        conn = self._get_conn()
+        try:
+            rows = conn.execute(
+                "SELECT id, event_type, actor, resource_id, details, "
+                "prev_hash, entry_hash, created_at "
+                "FROM audit_events ORDER BY id"
+            ).fetchall()
+        finally:
+            conn.close()
+
+        if not rows:
+            return {"valid": True, "entries_checked": 0, "error": None}
+
+        expected_prev = _GENESIS
+        for row in rows:
+            row = dict(row)
+            # Check the prev_hash link
+            if row["prev_hash"] != expected_prev:
+                return {
+                    "valid": False,
+                    "entries_checked": row["id"],
+                    "error": f"prev_hash mismatch at entry {row['id']}",
+                }
+            # Recompute the entry hash
+            computed = _compute_hash(
+                row["event_type"],
+                row["actor"],
+                row["resource_id"],
+                row["details"],
+                row["prev_hash"],
+                row["created_at"],
+            )
+            if computed != row["entry_hash"]:
+                return {
+                    "valid": False,
+                    "entries_checked": row["id"],
+                    "error": f"entry_hash mismatch at entry {row['id']}",
+                }
+            expected_prev = row["entry_hash"]
+
+        return {
+            "valid": True,
+            "entries_checked": len(rows),
+            "error": None,
+        }

package/src/compliance/audit_logger.py

@@ -0,0 +1,148 @@
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
+"""EventBus listener that writes all events to audit.db.
+
+Bridges the EventBus (real-time event emission) with AuditDB (tamper-evident
+audit trail). Every event that passes through the EventBus gets persisted
+into audit.db with full hash-chain integrity.
+
+Thread-safe: handle_event() runs on the emitter's thread and must be fast.
+Graceful: malformed events are logged defensively, never crash the caller.
+"""
+import json
+import logging
+import threading
+from typing import Any, Dict, Optional
+
+from .audit_db import AuditDB
+
+logger = logging.getLogger("superlocalmemory.compliance.audit_logger")
+
+
+class AuditLogger:
+    """Listens to EventBus events and writes them to audit.db.
+
+    Usage:
+        audit_logger = AuditLogger("/path/to/audit.db")
+        audit_logger.register_with_eventbus()  # auto-subscribe
+
+    Or manually:
+        event_bus.add_listener(audit_logger.handle_event)
+    """
+
+    def __init__(self, audit_db_path: str):
+        self._audit_db = AuditDB(audit_db_path)
+        self._lock = threading.Lock()
+        self._events_logged: int = 0
+        self._errors: int = 0
+        self._registered: bool = False
+
+    # ------------------------------------------------------------------
+    # Public API
+    # ------------------------------------------------------------------
+
+    @property
+    def events_logged(self) -> int:
+        """Total number of events successfully written to audit.db."""
+        return self._events_logged
+
+    def handle_event(self, event: Dict[str, Any]) -> None:
+        """Process a single EventBus event and write it to audit.db.
+
+        Extracts event_type, source_agent (actor), memory_id (resource_id),
+        and payload (details) from the event dict, then delegates to
+        AuditDB.log_event().
+
+        This method MUST NOT raise — it runs on the emitter's thread.
+        Any failure is caught, logged, and counted in self._errors.
+
+        Args:
+            event: Dict emitted by EventBus. Expected keys:
+                   event_type, source_agent, memory_id, payload, timestamp.
+                   All keys are optional for graceful degradation.
+        """
+        try:
+            if not isinstance(event, dict):
+                logger.warning("AuditLogger received non-dict event: %s", type(event))
+                return
+
+            event_type = event.get("event_type", "unknown")
+            actor = event.get("source_agent", "system")
+            resource_id = event.get("memory_id")
+            payload = event.get("payload", {})
+
+            # Build details dict including any extra context
+            details = {}
+            if isinstance(payload, dict):
+                details.update(payload)
+            else:
+                details["raw_payload"] = str(payload)
+
+            # Include timestamp from event if present
+            ts = event.get("timestamp")
+            if ts:
+                details["event_timestamp"] = ts
+
+            with self._lock:
+                self._audit_db.log_event(
+                    event_type=event_type,
+                    actor=actor,
+                    resource_id=resource_id,
+                    details=details,
+                )
+                self._events_logged += 1
+
+        except Exception as exc:
+            self._errors += 1
+            logger.error(
+                "AuditLogger failed to log event: %s (event=%s)",
+                exc,
+                _safe_repr(event),
+            )
+
+    def register_with_eventbus(self) -> bool:
+        """Register this logger as an EventBus listener.
+
+        Attempts to find the EventBus singleton and subscribe
+        handle_event as a listener. Returns True on success,
+        False if EventBus is unavailable.
+
+        Graceful: never raises; returns False on any failure.
+        """
+        try:
+            from event_bus import EventBus as EB
+
+            bus = EB.get_instance()
+            bus.add_listener(self.handle_event)
+            self._registered = True
+            logger.info("AuditLogger registered with EventBus")
+            return True
+        except Exception as exc:
+            logger.warning("AuditLogger could not register with EventBus: %s", exc)
+            self._registered = False
+            return False
+
+    def get_status(self) -> Dict[str, Any]:
+        """Return diagnostic status of this audit logger.
+
+        Returns:
+            Dict with keys: events_logged, errors, registered.
+        """
+        return {
+            "events_logged": self._events_logged,
+            "errors": self._errors,
+            "registered": self._registered,
+        }
+
+
+# ------------------------------------------------------------------
+# Internal helpers
+# ------------------------------------------------------------------
+
+def _safe_repr(obj: Any, max_len: int = 200) -> str:
+    """Safe repr that truncates and never raises."""
+    try:
+        r = repr(obj)
+        return r[:max_len] + "..." if len(r) > max_len else r
+    except Exception:
+        return "<unrepresentable>"

package/src/compliance/retention_manager.py

@@ -0,0 +1,289 @@
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
+"""Compliance retention manager — regulatory retention enforcement.
+
+Unlike the lifecycle ``retention_policy.py`` (which manages lifecycle-level
+policies stored alongside memory.db), this compliance module is the
+*regulatory* layer that:
+
+- Links retention rules to regulatory frameworks (GDPR, EU AI Act, HIPAA).
+- Enforces GDPR right-to-erasure (tombstone memory + preserve audit trail).
+- Enforces EU AI Act audit retention (10-year minimum for audit records).
+- Records every retention action in audit.db for tamper-evident compliance.
+
+Rules are stored in audit.db (``compliance_retention_rules`` table) so that
+the audit database remains the single source of truth for all compliance
+configuration and evidence.
+"""
+import hashlib
+import json
+import logging
+import sqlite3
+from datetime import datetime, timezone
+from typing import Any, Dict, List, Optional
+
+logger = logging.getLogger(__name__)
+
+_RULES_TABLE_SQL = """
+CREATE TABLE IF NOT EXISTS compliance_retention_rules (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    name TEXT NOT NULL,
+    framework TEXT NOT NULL,
+    retention_days INTEGER NOT NULL,
+    action TEXT NOT NULL,
+    applies_to TEXT NOT NULL,
+    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+)
+"""
+
+_AUDIT_EVENTS_TABLE_SQL = """
+CREATE TABLE IF NOT EXISTS audit_events (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    event_type TEXT NOT NULL,
+    actor TEXT NOT NULL,
+    resource_id INTEGER,
+    details TEXT DEFAULT '{}',
+    prev_hash TEXT NOT NULL DEFAULT 'genesis',
+    entry_hash TEXT NOT NULL DEFAULT '',
+    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+)
+"""
+
+
+def _compute_hash(event_type: str, actor: str, resource_id: Any,
+                  details: str, prev_hash: str, ts: str) -> str:
+    """Compute a SHA-256 hash for a single audit event."""
+    payload = f"{event_type}|{actor}|{resource_id}|{details}|{prev_hash}|{ts}"
+    return hashlib.sha256(payload.encode("utf-8")).hexdigest()
+
+
+class ComplianceRetentionManager:
+    """Enforces regulatory retention policies across memory and audit DBs.
+
+    Connects to *both* databases:
+    - ``memory_db_path``: where memories live (tombstoning happens here).
+    - ``audit_db_path``: where rules and audit events are stored.
+    """
+
+    def __init__(self, memory_db_path: str, audit_db_path: str):
+        self._memory_db_path = memory_db_path
+        self._audit_db_path = audit_db_path
+        self._ensure_tables()
+
+    # ------------------------------------------------------------------
+    # Internal helpers
+    # ------------------------------------------------------------------
+
+    def _connect_audit(self) -> sqlite3.Connection:
+        conn = sqlite3.connect(self._audit_db_path)
+        conn.row_factory = sqlite3.Row
+        return conn
+
+    def _connect_memory(self) -> sqlite3.Connection:
+        conn = sqlite3.connect(self._memory_db_path)
+        conn.row_factory = sqlite3.Row
+        return conn
+
+    def _ensure_tables(self) -> None:
+        conn = self._connect_audit()
+        try:
+            conn.execute(_RULES_TABLE_SQL)
+            conn.execute(_AUDIT_EVENTS_TABLE_SQL)
+            conn.commit()
+        finally:
+            conn.close()
+
+    def _log_audit_event(self, event_type: str, actor: str,
+                         resource_id: Optional[int],
+                         details: Dict[str, Any]) -> None:
+        """Append a tamper-evident audit event to audit.db."""
+        conn = self._connect_audit()
+        try:
+            last = conn.execute(
+                "SELECT entry_hash FROM audit_events ORDER BY id DESC LIMIT 1"
+            ).fetchone()
+            prev_hash = last["entry_hash"] if last else "genesis"
+            ts = datetime.now(timezone.utc).isoformat()
+            details_json = json.dumps(details, default=str)
+            entry_hash = _compute_hash(
+                event_type, actor, resource_id, details_json, prev_hash, ts,
+            )
+            conn.execute(
+                "INSERT INTO audit_events "
+                "(event_type, actor, resource_id, details, prev_hash, entry_hash, created_at) "
+                "VALUES (?, ?, ?, ?, ?, ?, ?)",
+                (event_type, actor, resource_id, details_json, prev_hash,
+                 entry_hash, ts),
+            )
+            conn.commit()
+        finally:
+            conn.close()
+
+    @staticmethod
+    def _parse_json(value: Any) -> Any:
+        if isinstance(value, str):
+            try:
+                return json.loads(value)
+            except (json.JSONDecodeError, TypeError):
+                return value
+        return value if value is not None else []
+
+    @staticmethod
+    def _matches(criteria: Any, mem_tags: Any, mem_project: Optional[str]) -> bool:
+        """Return True when a rule's ``applies_to`` matches the memory."""
+        if not isinstance(criteria, dict) or not criteria:
+            return False
+        ok = True
+        if "tags" in criteria:
+            rule_tags = set(criteria["tags"]) if criteria["tags"] else set()
+            m_tags = set(mem_tags) if isinstance(mem_tags, list) else set()
+            if not rule_tags & m_tags:
+                ok = False
+        if "project_name" in criteria:
+            if mem_project != criteria["project_name"]:
+                ok = False
+        return ok
+
+    # ------------------------------------------------------------------
+    # Public API
+    # ------------------------------------------------------------------
+
+    def create_retention_rule(self, name: str, framework: str,
+                              retention_days: int, action: str,
+                              applies_to: Dict[str, Any]) -> int:
+        """Create a compliance retention rule in audit.db.
+
+        Returns the auto-generated rule ID.
+        """
+        conn = self._connect_audit()
+        try:
+            cur = conn.execute(
+                "INSERT INTO compliance_retention_rules "
+                "(name, framework, retention_days, action, applies_to) "
+                "VALUES (?, ?, ?, ?, ?)",
+                (name, framework, retention_days, action,
+                 json.dumps(applies_to)),
+            )
+            conn.commit()
+            rule_id = cur.lastrowid
+        finally:
+            conn.close()
+
+        self._log_audit_event(
+            "retention.rule_created", "system", rule_id,
+            {"name": name, "framework": framework},
+        )
+        return rule_id
+
+    def list_rules(self) -> List[Dict[str, Any]]:
+        """Return all compliance retention rules."""
+        conn = self._connect_audit()
+        try:
+            rows = conn.execute(
+                "SELECT * FROM compliance_retention_rules ORDER BY id"
+            ).fetchall()
+            result = []
+            for r in rows:
+                d = dict(r)
+                if isinstance(d.get("applies_to"), str):
+                    d["applies_to"] = self._parse_json(d["applies_to"])
+                result.append(d)
+            return result
+        finally:
+            conn.close()
+
+    def evaluate_memory(self, memory_id: int) -> Optional[Dict[str, Any]]:
+        """Check which compliance rule applies to a memory.
+
+        Reads the memory's tags/project from memory.db, then evaluates
+        all rules from audit.db. The first matching rule (ordered by id)
+        is returned.
+
+        Returns a dict with ``rule_name``, ``action``, ``retention_days``,
+        ``framework``; or ``None`` if no rule matches.
+        """
+        mem_conn = self._connect_memory()
+        try:
+            mem = mem_conn.execute(
+                "SELECT tags, project_name FROM memories WHERE id = ?",
+                (memory_id,),
+            ).fetchone()
+            if mem is None:
+                return None
+            mem_tags = self._parse_json(mem["tags"])
+            mem_project = mem["project_name"]
+        finally:
+            mem_conn.close()
+
+        audit_conn = self._connect_audit()
+        try:
+            rules = audit_conn.execute(
+                "SELECT * FROM compliance_retention_rules ORDER BY id"
+            ).fetchall()
+            for rule in rules:
+                criteria = self._parse_json(rule["applies_to"])
+                if self._matches(criteria, mem_tags, mem_project):
+                    return {
+                        "rule_name": rule["name"],
+                        "action": rule["action"],
+                        "retention_days": rule["retention_days"],
+                        "framework": rule["framework"],
+                    }
+            return None
+        finally:
+            audit_conn.close()
+
+    def execute_erasure_request(self, memory_id: int, framework: str,
+                                requested_by: str) -> Dict[str, Any]:
+        """Execute a GDPR (or other framework) right-to-erasure request.
+
+        1. Tombstones the memory in memory.db.
+        2. Logs the erasure event in audit.db (preserving the audit trail).
+
+        Returns a result dict with ``success``, ``action``, and ``memory_id``.
+        """
+        mem_conn = self._connect_memory()
+        try:
+            row = mem_conn.execute(
+                "SELECT id FROM memories WHERE id = ?", (memory_id,),
+            ).fetchone()
+            if row is None:
+                return {"success": False, "error": "memory_not_found",
+                        "memory_id": memory_id}
+
+            ts = datetime.now(timezone.utc).isoformat()
+            mem_conn.execute(
+                "UPDATE memories SET lifecycle_state = 'tombstoned', "
+                "lifecycle_updated_at = ? WHERE id = ?",
+                (ts, memory_id),
+            )
+            mem_conn.commit()
+        finally:
+            mem_conn.close()
+
+        self._log_audit_event(
+            "retention.erasure", requested_by, memory_id,
+            {"framework": framework, "action": "tombstoned"},
+        )
+
+        return {"success": True, "action": "tombstoned",
+                "memory_id": memory_id}
+
+    def get_compliance_status(self) -> Dict[str, Any]:
+        """Return a summary of current compliance retention state."""
+        conn = self._connect_audit()
+        try:
+            rules = conn.execute(
+                "SELECT * FROM compliance_retention_rules"
+            ).fetchall()
+            frameworks = list({r["framework"] for r in rules})
+            events_count = conn.execute(
+                "SELECT COUNT(*) AS cnt FROM audit_events"
+            ).fetchone()["cnt"]
+            return {
+                "rules_count": len(rules),
+                "frameworks": sorted(frameworks),
+                "audit_events_count": events_count,
+            }
+        finally:
+            conn.close()