superlocalmemory 2.7.6 → 2.8.1

package/src/compliance/retention_manager.py

@@ -0,0 +1,289 @@
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
+"""Compliance retention manager — regulatory retention enforcement.
+
+Unlike the lifecycle ``retention_policy.py`` (which manages lifecycle-level
+policies stored alongside memory.db), this compliance module is the
+*regulatory* layer that:
+
+- Links retention rules to regulatory frameworks (GDPR, EU AI Act, HIPAA).
+- Enforces GDPR right-to-erasure (tombstone memory + preserve audit trail).
+- Enforces EU AI Act audit retention (10-year minimum for audit records).
+- Records every retention action in audit.db for tamper-evident compliance.
+
+Rules are stored in audit.db (``compliance_retention_rules`` table) so that
+the audit database remains the single source of truth for all compliance
+configuration and evidence.
+"""
+import hashlib
+import json
+import logging
+import sqlite3
+from datetime import datetime, timezone
+from typing import Any, Dict, List, Optional
+
+logger = logging.getLogger(__name__)
+
+_RULES_TABLE_SQL = """
+CREATE TABLE IF NOT EXISTS compliance_retention_rules (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    name TEXT NOT NULL,
+    framework TEXT NOT NULL,
+    retention_days INTEGER NOT NULL,
+    action TEXT NOT NULL,
+    applies_to TEXT NOT NULL,
+    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+)
+"""
+
+_AUDIT_EVENTS_TABLE_SQL = """
+CREATE TABLE IF NOT EXISTS audit_events (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    event_type TEXT NOT NULL,
+    actor TEXT NOT NULL,
+    resource_id INTEGER,
+    details TEXT DEFAULT '{}',
+    prev_hash TEXT NOT NULL DEFAULT 'genesis',
+    entry_hash TEXT NOT NULL DEFAULT '',
+    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+)
+"""
+
+
+def _compute_hash(event_type: str, actor: str, resource_id: Any,
+                  details: str, prev_hash: str, ts: str) -> str:
+    """Compute a SHA-256 hash for a single audit event."""
+    payload = f"{event_type}|{actor}|{resource_id}|{details}|{prev_hash}|{ts}"
+    return hashlib.sha256(payload.encode("utf-8")).hexdigest()
+
+
+class ComplianceRetentionManager:
+    """Enforces regulatory retention policies across memory and audit DBs.
+
+    Connects to *both* databases:
+    - ``memory_db_path``: where memories live (tombstoning happens here).
+    - ``audit_db_path``: where rules and audit events are stored.
+    """
+
+    def __init__(self, memory_db_path: str, audit_db_path: str):
+        self._memory_db_path = memory_db_path
+        self._audit_db_path = audit_db_path
+        self._ensure_tables()
+
+    # ------------------------------------------------------------------
+    # Internal helpers
+    # ------------------------------------------------------------------
+
+    def _connect_audit(self) -> sqlite3.Connection:
+        conn = sqlite3.connect(self._audit_db_path)
+        conn.row_factory = sqlite3.Row
+        return conn
+
+    def _connect_memory(self) -> sqlite3.Connection:
+        conn = sqlite3.connect(self._memory_db_path)
+        conn.row_factory = sqlite3.Row
+        return conn
+
+    def _ensure_tables(self) -> None:
+        conn = self._connect_audit()
+        try:
+            conn.execute(_RULES_TABLE_SQL)
+            conn.execute(_AUDIT_EVENTS_TABLE_SQL)
+            conn.commit()
+        finally:
+            conn.close()
+
+    def _log_audit_event(self, event_type: str, actor: str,
+                         resource_id: Optional[int],
+                         details: Dict[str, Any]) -> None:
+        """Append a tamper-evident audit event to audit.db."""
+        conn = self._connect_audit()
+        try:
+            last = conn.execute(
+                "SELECT entry_hash FROM audit_events ORDER BY id DESC LIMIT 1"
+            ).fetchone()
+            prev_hash = last["entry_hash"] if last else "genesis"
+            ts = datetime.now(timezone.utc).isoformat()
+            details_json = json.dumps(details, default=str)
+            entry_hash = _compute_hash(
+                event_type, actor, resource_id, details_json, prev_hash, ts,
+            )
+            conn.execute(
+                "INSERT INTO audit_events "
+                "(event_type, actor, resource_id, details, prev_hash, entry_hash, created_at) "
+                "VALUES (?, ?, ?, ?, ?, ?, ?)",
+                (event_type, actor, resource_id, details_json, prev_hash,
+                 entry_hash, ts),
+            )
+            conn.commit()
+        finally:
+            conn.close()
+
+    @staticmethod
+    def _parse_json(value: Any) -> Any:
+        if isinstance(value, str):
+            try:
+                return json.loads(value)
+            except (json.JSONDecodeError, TypeError):
+                return value
+        return value if value is not None else []
+
+    @staticmethod
+    def _matches(criteria: Any, mem_tags: Any, mem_project: Optional[str]) -> bool:
+        """Return True when a rule's ``applies_to`` matches the memory."""
+        if not isinstance(criteria, dict) or not criteria:
+            return False
+        ok = True
+        if "tags" in criteria:
+            rule_tags = set(criteria["tags"]) if criteria["tags"] else set()
+            m_tags = set(mem_tags) if isinstance(mem_tags, list) else set()
+            if not rule_tags & m_tags:
+                ok = False
+        if "project_name" in criteria:
+            if mem_project != criteria["project_name"]:
+                ok = False
+        return ok
+
+    # ------------------------------------------------------------------
+    # Public API
+    # ------------------------------------------------------------------
+
+    def create_retention_rule(self, name: str, framework: str,
+                              retention_days: int, action: str,
+                              applies_to: Dict[str, Any]) -> int:
+        """Create a compliance retention rule in audit.db.
+
+        Returns the auto-generated rule ID.
+        """
+        conn = self._connect_audit()
+        try:
+            cur = conn.execute(
+                "INSERT INTO compliance_retention_rules "
+                "(name, framework, retention_days, action, applies_to) "
+                "VALUES (?, ?, ?, ?, ?)",
+                (name, framework, retention_days, action,
+                 json.dumps(applies_to)),
+            )
+            conn.commit()
+            rule_id = cur.lastrowid
+        finally:
+            conn.close()
+
+        self._log_audit_event(
+            "retention.rule_created", "system", rule_id,
+            {"name": name, "framework": framework},
+        )
+        return rule_id
+
+    def list_rules(self) -> List[Dict[str, Any]]:
+        """Return all compliance retention rules."""
+        conn = self._connect_audit()
+        try:
+            rows = conn.execute(
+                "SELECT * FROM compliance_retention_rules ORDER BY id"
+            ).fetchall()
+            result = []
+            for r in rows:
+                d = dict(r)
+                if isinstance(d.get("applies_to"), str):
+                    d["applies_to"] = self._parse_json(d["applies_to"])
+                result.append(d)
+            return result
+        finally:
+            conn.close()
+
+    def evaluate_memory(self, memory_id: int) -> Optional[Dict[str, Any]]:
+        """Check which compliance rule applies to a memory.
+
+        Reads the memory's tags/project from memory.db, then evaluates
+        all rules from audit.db. The first matching rule (ordered by id)
+        is returned.
+
+        Returns a dict with ``rule_name``, ``action``, ``retention_days``,
+        ``framework``; or ``None`` if no rule matches.
+        """
+        mem_conn = self._connect_memory()
+        try:
+            mem = mem_conn.execute(
+                "SELECT tags, project_name FROM memories WHERE id = ?",
+                (memory_id,),
+            ).fetchone()
+            if mem is None:
+                return None
+            mem_tags = self._parse_json(mem["tags"])
+            mem_project = mem["project_name"]
+        finally:
+            mem_conn.close()
+
+        audit_conn = self._connect_audit()
+        try:
+            rules = audit_conn.execute(
+                "SELECT * FROM compliance_retention_rules ORDER BY id"
+            ).fetchall()
+            for rule in rules:
+                criteria = self._parse_json(rule["applies_to"])
+                if self._matches(criteria, mem_tags, mem_project):
+                    return {
+                        "rule_name": rule["name"],
+                        "action": rule["action"],
+                        "retention_days": rule["retention_days"],
+                        "framework": rule["framework"],
+                    }
+            return None
+        finally:
+            audit_conn.close()
+
+    def execute_erasure_request(self, memory_id: int, framework: str,
+                                requested_by: str) -> Dict[str, Any]:
+        """Execute a GDPR (or other framework) right-to-erasure request.
+
+        1. Tombstones the memory in memory.db.
+        2. Logs the erasure event in audit.db (preserving the audit trail).
+
+        Returns a result dict with ``success``, ``action``, and ``memory_id``.
+        """
+        mem_conn = self._connect_memory()
+        try:
+            row = mem_conn.execute(
+                "SELECT id FROM memories WHERE id = ?", (memory_id,),
+            ).fetchone()
+            if row is None:
+                return {"success": False, "error": "memory_not_found",
+                        "memory_id": memory_id}
+
+            ts = datetime.now(timezone.utc).isoformat()
+            mem_conn.execute(
+                "UPDATE memories SET lifecycle_state = 'tombstoned', "
+                "lifecycle_updated_at = ? WHERE id = ?",
+                (ts, memory_id),
+            )
+            mem_conn.commit()
+        finally:
+            mem_conn.close()
+
+        self._log_audit_event(
+            "retention.erasure", requested_by, memory_id,
+            {"framework": framework, "action": "tombstoned"},
+        )
+
+        return {"success": True, "action": "tombstoned",
+                "memory_id": memory_id}
+
+    def get_compliance_status(self) -> Dict[str, Any]:
+        """Return a summary of current compliance retention state."""
+        conn = self._connect_audit()
+        try:
+            rules = conn.execute(
+                "SELECT * FROM compliance_retention_rules"
+            ).fetchall()
+            frameworks = list({r["framework"] for r in rules})
+            events_count = conn.execute(
+                "SELECT COUNT(*) AS cnt FROM audit_events"
+            ).fetchone()["cnt"]
+            return {
+                "rules_count": len(rules),
+                "frameworks": sorted(frameworks),
+                "audit_events_count": events_count,
+            }
+        finally:
+            conn.close()

package/src/compliance/retention_scheduler.py

@@ -0,0 +1,186 @@
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
+"""Background scheduler for periodic retention policy enforcement.
+
+Runs on a configurable interval (default: 24 hours) to:
+1. Load compliance retention rules from audit.db
+2. Scan all memories in memory.db against those rules
+3. Tombstone expired memories (age exceeds retention_days)
+4. Log every action to audit.db for tamper-evident compliance
+
+Uses daemon threading -- does not prevent process exit.
+"""
+import json
+import logging
+import sqlite3
+import threading
+from datetime import datetime, timezone
+from typing import Any, Dict, List, Optional
+
+from .retention_manager import ComplianceRetentionManager
+
+logger = logging.getLogger(__name__)
+
+# Default interval: 24 hours
+DEFAULT_INTERVAL_SECONDS = 86400
+
+
+class RetentionScheduler:
+    """Background scheduler for periodic retention policy enforcement.
+
+    Orchestrates ComplianceRetentionManager on a configurable timer
+    interval to automatically enforce retention rules.
+    """
+
+    def __init__(
+        self,
+        memory_db_path: str,
+        audit_db_path: str,
+        interval_seconds: int = DEFAULT_INTERVAL_SECONDS,
+    ):
+        self._memory_db_path = memory_db_path
+        self._audit_db_path = audit_db_path
+        self.interval_seconds = interval_seconds
+
+        self._manager = ComplianceRetentionManager(
+            memory_db_path, audit_db_path,
+        )
+
+        self._timer: Optional[threading.Timer] = None
+        self._running = False
+        self._lock = threading.Lock()
+
+    @property
+    def is_running(self) -> bool:
+        """Whether the scheduler is currently running."""
+        return self._running
+
+    def start(self) -> None:
+        """Start the background scheduler."""
+        with self._lock:
+            if self._running:
+                return
+            self._running = True
+            self._schedule_next()
+
+    def stop(self) -> None:
+        """Stop the background scheduler."""
+        with self._lock:
+            self._running = False
+            if self._timer is not None:
+                self._timer.cancel()
+                self._timer = None
+
+    def run_now(self) -> Dict[str, Any]:
+        """Execute a retention enforcement cycle immediately.
+
+        Returns:
+            Dict with timestamp, actions taken, and rules evaluated.
+        """
+        return self._execute_cycle()
+
+    def _schedule_next(self) -> None:
+        """Schedule the next enforcement cycle."""
+        self._timer = threading.Timer(self.interval_seconds, self._run_cycle)
+        self._timer.daemon = True
+        self._timer.start()
+
+    def _run_cycle(self) -> None:
+        """Run one enforcement cycle, then schedule the next."""
+        try:
+            self._execute_cycle()
+        except Exception:
+            pass  # Scheduler must not crash
+        finally:
+            with self._lock:
+                if self._running:
+                    self._schedule_next()
+
+    def _execute_cycle(self) -> Dict[str, Any]:
+        """Core retention enforcement logic.
+
+        1. Load all retention rules from audit.db
+        2. Scan every memory against each rule
+        3. Tombstone memories that exceed retention_days
+        4. Log actions to audit.db
+        """
+        rules = self._manager.list_rules()
+        actions: List[Dict[str, Any]] = []
+
+        # Scan all memories
+        memory_ids = self._get_all_memory_ids()
+
+        for mem_id in memory_ids:
+            mem = self._get_memory(mem_id)
+            if mem is None:
+                continue
+
+            # Already tombstoned -- skip
+            if mem.get("lifecycle_state") == "tombstoned":
+                continue
+
+            match = self._manager.evaluate_memory(mem_id)
+            if match is None:
+                continue
+
+            # Check if memory age exceeds the rule's retention_days
+            created_at = mem.get("created_at")
+            if created_at is None:
+                continue
+
+            age_days = self._age_in_days(created_at)
+            if age_days > match["retention_days"]:
+                action = match["action"]
+                if action == "tombstone":
+                    result = self._manager.execute_erasure_request(
+                        mem_id, match["framework"], "retention_scheduler",
+                    )
+                    actions.append({
+                        "memory_id": mem_id,
+                        "action": action,
+                        "rule_name": match["rule_name"],
+                        "framework": match["framework"],
+                        "age_days": age_days,
+                        "success": result.get("success", False),
+                    })
+
+        return {
+            "timestamp": datetime.now(timezone.utc).isoformat(),
+            "actions": actions,
+            "rules_evaluated": len(rules),
+        }
+
+    # ------------------------------------------------------------------
+    # Internal helpers
+    # ------------------------------------------------------------------
+
+    def _get_all_memory_ids(self) -> List[int]:
+        """Return all memory IDs from memory.db."""
+        conn = sqlite3.connect(self._memory_db_path)
+        try:
+            rows = conn.execute("SELECT id FROM memories").fetchall()
+            return [r[0] for r in rows]
+        finally:
+            conn.close()
+
+    def _get_memory(self, memory_id: int) -> Optional[Dict[str, Any]]:
+        """Fetch a single memory row as a dict."""
+        conn = sqlite3.connect(self._memory_db_path)
+        conn.row_factory = sqlite3.Row
+        try:
+            row = conn.execute(
+                "SELECT * FROM memories WHERE id = ?", (memory_id,),
+            ).fetchone()
+            return dict(row) if row else None
+        finally:
+            conn.close()
+
+    @staticmethod
+    def _age_in_days(created_at_str: str) -> float:
+        """Calculate age of a memory in days from its created_at."""
+        try:
+            created = datetime.fromisoformat(created_at_str)
+            now = datetime.now(created.tzinfo)
+            return (now - created).total_seconds() / 86400
+        except (ValueError, TypeError):
+            return 0.0

package/src/compliance/tests/__init__.py

@@ -0,0 +1,4 @@
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
+"""Tests for compliance engine.
+"""

package/src/compliance/tests/test_abac_enforcement.py

@@ -0,0 +1,95 @@
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
+"""Tests for ABAC enforcement in memory operations.
+"""
+import sqlite3
+import tempfile
+import os
+import sys
+import json
+import shutil
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent))
+
+
+class TestABACEnforcement:
+    def setup_method(self):
+        self.tmp_dir = tempfile.mkdtemp()
+        self.db_path = os.path.join(self.tmp_dir, "memory.db")
+        # Create store with test data
+        from memory_store_v2 import MemoryStoreV2
+        self.store = MemoryStoreV2(self.db_path)
+        self.store.add_memory(
+            content="public memory about Python",
+            tags=["python"],
+            importance=5,
+        )
+        self.store.add_memory(
+            content="private memory about secrets",
+            tags=["secrets"],
+            importance=8,
+        )
+        # Set access_level for private memory
+        conn = sqlite3.connect(self.db_path)
+        conn.execute("UPDATE memories SET access_level='private' WHERE id=2")
+        conn.commit()
+        conn.close()
+        self.store._rebuild_vectors()
+
+    def teardown_method(self):
+        shutil.rmtree(self.tmp_dir, ignore_errors=True)
+
+    def test_search_without_abac_works(self):
+        """Search without ABAC context works (backward compat)."""
+        results = self.store.search("Python", limit=5)
+        assert len(results) >= 1
+
+    def test_search_with_agent_context(self):
+        """Search with agent_context parameter works."""
+        results = self.store.search(
+            "memory", limit=10, agent_context={"agent_id": "user"}
+        )
+        assert isinstance(results, list)
+
+    def test_create_without_abac_works(self):
+        """Create without ABAC works (backward compat)."""
+        mem_id = self.store.add_memory(content="new memory", tags=["test"])
+        assert mem_id is not None
+
+    def test_abac_check_method_exists(self):
+        """MemoryStoreV2 has _check_abac method."""
+        assert hasattr(self.store, "_check_abac")
+
+    def test_check_abac_default_allows(self):
+        """Default ABAC check (no policy file) allows everything."""
+        result = self.store._check_abac(
+            subject={"agent_id": "user"},
+            resource={"access_level": "public"},
+            action="read",
+        )
+        assert result["allowed"] is True
+
+    def test_check_abac_with_policy(self):
+        """ABAC check with policy file respects deny rules."""
+        policy_path = os.path.join(self.tmp_dir, "abac_policies.json")
+        with open(policy_path, "w") as f:
+            json.dump(
+                [
+                    {
+                        "name": "deny-private",
+                        "effect": "deny",
+                        "subjects": {"agent_id": "*"},
+                        "resources": {"access_level": "private"},
+                        "actions": ["read"],
+                    }
+                ],
+                f,
+            )
+        result = self.store._check_abac(
+            subject={"agent_id": "bot"},
+            resource={"access_level": "private"},
+            action="read",
+            policy_path=policy_path,
+        )
+        assert result["allowed"] is False