superlocalmemory 2.8.5 → 3.0.0

package/src/compliance/retention_manager.py

@@ -1,289 +0,0 @@
-# SPDX-License-Identifier: MIT
-# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
-"""Compliance retention manager — regulatory retention enforcement.
-
-Unlike the lifecycle ``retention_policy.py`` (which manages lifecycle-level
-policies stored alongside memory.db), this compliance module is the
-*regulatory* layer that:
-
-- Links retention rules to regulatory frameworks (GDPR, EU AI Act, HIPAA).
-- Enforces GDPR right-to-erasure (tombstone memory + preserve audit trail).
-- Enforces EU AI Act audit retention (10-year minimum for audit records).
-- Records every retention action in audit.db for tamper-evident compliance.
-
-Rules are stored in audit.db (``compliance_retention_rules`` table) so that
-the audit database remains the single source of truth for all compliance
-configuration and evidence.
-"""
-import hashlib
-import json
-import logging
-import sqlite3
-from datetime import datetime, timezone
-from typing import Any, Dict, List, Optional
-
-logger = logging.getLogger(__name__)
-
-_RULES_TABLE_SQL = """
-CREATE TABLE IF NOT EXISTS compliance_retention_rules (
-    id INTEGER PRIMARY KEY AUTOINCREMENT,
-    name TEXT NOT NULL,
-    framework TEXT NOT NULL,
-    retention_days INTEGER NOT NULL,
-    action TEXT NOT NULL,
-    applies_to TEXT NOT NULL,
-    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
-)
-"""
-
-_AUDIT_EVENTS_TABLE_SQL = """
-CREATE TABLE IF NOT EXISTS audit_events (
-    id INTEGER PRIMARY KEY AUTOINCREMENT,
-    event_type TEXT NOT NULL,
-    actor TEXT NOT NULL,
-    resource_id INTEGER,
-    details TEXT DEFAULT '{}',
-    prev_hash TEXT NOT NULL DEFAULT 'genesis',
-    entry_hash TEXT NOT NULL DEFAULT '',
-    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
-)
-"""
-
-
-def _compute_hash(event_type: str, actor: str, resource_id: Any,
-                  details: str, prev_hash: str, ts: str) -> str:
-    """Compute a SHA-256 hash for a single audit event."""
-    payload = f"{event_type}|{actor}|{resource_id}|{details}|{prev_hash}|{ts}"
-    return hashlib.sha256(payload.encode("utf-8")).hexdigest()
-
-
-class ComplianceRetentionManager:
-    """Enforces regulatory retention policies across memory and audit DBs.
-
-    Connects to *both* databases:
-    - ``memory_db_path``: where memories live (tombstoning happens here).
-    - ``audit_db_path``: where rules and audit events are stored.
-    """
-
-    def __init__(self, memory_db_path: str, audit_db_path: str):
-        self._memory_db_path = memory_db_path
-        self._audit_db_path = audit_db_path
-        self._ensure_tables()
-
-    # ------------------------------------------------------------------
-    # Internal helpers
-    # ------------------------------------------------------------------
-
-    def _connect_audit(self) -> sqlite3.Connection:
-        conn = sqlite3.connect(self._audit_db_path)
-        conn.row_factory = sqlite3.Row
-        return conn
-
-    def _connect_memory(self) -> sqlite3.Connection:
-        conn = sqlite3.connect(self._memory_db_path)
-        conn.row_factory = sqlite3.Row
-        return conn
-
-    def _ensure_tables(self) -> None:
-        conn = self._connect_audit()
-        try:
-            conn.execute(_RULES_TABLE_SQL)
-            conn.execute(_AUDIT_EVENTS_TABLE_SQL)
-            conn.commit()
-        finally:
-            conn.close()
-
-    def _log_audit_event(self, event_type: str, actor: str,
-                         resource_id: Optional[int],
-                         details: Dict[str, Any]) -> None:
-        """Append a tamper-evident audit event to audit.db."""
-        conn = self._connect_audit()
-        try:
-            last = conn.execute(
-                "SELECT entry_hash FROM audit_events ORDER BY id DESC LIMIT 1"
-            ).fetchone()
-            prev_hash = last["entry_hash"] if last else "genesis"
-            ts = datetime.now(timezone.utc).isoformat()
-            details_json = json.dumps(details, default=str)
-            entry_hash = _compute_hash(
-                event_type, actor, resource_id, details_json, prev_hash, ts,
-            )
-            conn.execute(
-                "INSERT INTO audit_events "
-                "(event_type, actor, resource_id, details, prev_hash, entry_hash, created_at) "
-                "VALUES (?, ?, ?, ?, ?, ?, ?)",
-                (event_type, actor, resource_id, details_json, prev_hash,
-                 entry_hash, ts),
-            )
-            conn.commit()
-        finally:
-            conn.close()
-
-    @staticmethod
-    def _parse_json(value: Any) -> Any:
-        if isinstance(value, str):
-            try:
-                return json.loads(value)
-            except (json.JSONDecodeError, TypeError):
-                return value
-        return value if value is not None else []
-
-    @staticmethod
-    def _matches(criteria: Any, mem_tags: Any, mem_project: Optional[str]) -> bool:
-        """Return True when a rule's ``applies_to`` matches the memory."""
-        if not isinstance(criteria, dict) or not criteria:
-            return False
-        ok = True
-        if "tags" in criteria:
-            rule_tags = set(criteria["tags"]) if criteria["tags"] else set()
-            m_tags = set(mem_tags) if isinstance(mem_tags, list) else set()
-            if not rule_tags & m_tags:
-                ok = False
-        if "project_name" in criteria:
-            if mem_project != criteria["project_name"]:
-                ok = False
-        return ok
-
-    # ------------------------------------------------------------------
-    # Public API
-    # ------------------------------------------------------------------
-
-    def create_retention_rule(self, name: str, framework: str,
-                              retention_days: int, action: str,
-                              applies_to: Dict[str, Any]) -> int:
-        """Create a compliance retention rule in audit.db.
-
-        Returns the auto-generated rule ID.
-        """
-        conn = self._connect_audit()
-        try:
-            cur = conn.execute(
-                "INSERT INTO compliance_retention_rules "
-                "(name, framework, retention_days, action, applies_to) "
-                "VALUES (?, ?, ?, ?, ?)",
-                (name, framework, retention_days, action,
-                 json.dumps(applies_to)),
-            )
-            conn.commit()
-            rule_id = cur.lastrowid
-        finally:
-            conn.close()
-
-        self._log_audit_event(
-            "retention.rule_created", "system", rule_id,
-            {"name": name, "framework": framework},
-        )
-        return rule_id
-
-    def list_rules(self) -> List[Dict[str, Any]]:
-        """Return all compliance retention rules."""
-        conn = self._connect_audit()
-        try:
-            rows = conn.execute(
-                "SELECT * FROM compliance_retention_rules ORDER BY id"
-            ).fetchall()
-            result = []
-            for r in rows:
-                d = dict(r)
-                if isinstance(d.get("applies_to"), str):
-                    d["applies_to"] = self._parse_json(d["applies_to"])
-                result.append(d)
-            return result
-        finally:
-            conn.close()
-
-    def evaluate_memory(self, memory_id: int) -> Optional[Dict[str, Any]]:
-        """Check which compliance rule applies to a memory.
-
-        Reads the memory's tags/project from memory.db, then evaluates
-        all rules from audit.db. The first matching rule (ordered by id)
-        is returned.
-
-        Returns a dict with ``rule_name``, ``action``, ``retention_days``,
-        ``framework``; or ``None`` if no rule matches.
-        """
-        mem_conn = self._connect_memory()
-        try:
-            mem = mem_conn.execute(
-                "SELECT tags, project_name FROM memories WHERE id = ?",
-                (memory_id,),
-            ).fetchone()
-            if mem is None:
-                return None
-            mem_tags = self._parse_json(mem["tags"])
-            mem_project = mem["project_name"]
-        finally:
-            mem_conn.close()
-
-        audit_conn = self._connect_audit()
-        try:
-            rules = audit_conn.execute(
-                "SELECT * FROM compliance_retention_rules ORDER BY id"
-            ).fetchall()
-            for rule in rules:
-                criteria = self._parse_json(rule["applies_to"])
-                if self._matches(criteria, mem_tags, mem_project):
-                    return {
-                        "rule_name": rule["name"],
-                        "action": rule["action"],
-                        "retention_days": rule["retention_days"],
-                        "framework": rule["framework"],
-                    }
-            return None
-        finally:
-            audit_conn.close()
-
-    def execute_erasure_request(self, memory_id: int, framework: str,
-                                requested_by: str) -> Dict[str, Any]:
-        """Execute a GDPR (or other framework) right-to-erasure request.
-
-        1. Tombstones the memory in memory.db.
-        2. Logs the erasure event in audit.db (preserving the audit trail).
-
-        Returns a result dict with ``success``, ``action``, and ``memory_id``.
-        """
-        mem_conn = self._connect_memory()
-        try:
-            row = mem_conn.execute(
-                "SELECT id FROM memories WHERE id = ?", (memory_id,),
-            ).fetchone()
-            if row is None:
-                return {"success": False, "error": "memory_not_found",
-                        "memory_id": memory_id}
-
-            ts = datetime.now(timezone.utc).isoformat()
-            mem_conn.execute(
-                "UPDATE memories SET lifecycle_state = 'tombstoned', "
-                "lifecycle_updated_at = ? WHERE id = ?",
-                (ts, memory_id),
-            )
-            mem_conn.commit()
-        finally:
-            mem_conn.close()
-
-        self._log_audit_event(
-            "retention.erasure", requested_by, memory_id,
-            {"framework": framework, "action": "tombstoned"},
-        )
-
-        return {"success": True, "action": "tombstoned",
-                "memory_id": memory_id}
-
-    def get_compliance_status(self) -> Dict[str, Any]:
-        """Return a summary of current compliance retention state."""
-        conn = self._connect_audit()
-        try:
-            rules = conn.execute(
-                "SELECT * FROM compliance_retention_rules"
-            ).fetchall()
-            frameworks = list({r["framework"] for r in rules})
-            events_count = conn.execute(
-                "SELECT COUNT(*) AS cnt FROM audit_events"
-            ).fetchone()["cnt"]
-            return {
-                "rules_count": len(rules),
-                "frameworks": sorted(frameworks),
-                "audit_events_count": events_count,
-            }
-        finally:
-            conn.close()

package/src/compliance/retention_scheduler.py

@@ -1,186 +0,0 @@
-# SPDX-License-Identifier: MIT
-# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
-"""Background scheduler for periodic retention policy enforcement.
-
-Runs on a configurable interval (default: 24 hours) to:
-1. Load compliance retention rules from audit.db
-2. Scan all memories in memory.db against those rules
-3. Tombstone expired memories (age exceeds retention_days)
-4. Log every action to audit.db for tamper-evident compliance
-
-Uses daemon threading -- does not prevent process exit.
-"""
-import json
-import logging
-import sqlite3
-import threading
-from datetime import datetime, timezone
-from typing import Any, Dict, List, Optional
-
-from .retention_manager import ComplianceRetentionManager
-
-logger = logging.getLogger(__name__)
-
-# Default interval: 24 hours
-DEFAULT_INTERVAL_SECONDS = 86400
-
-
-class RetentionScheduler:
-    """Background scheduler for periodic retention policy enforcement.
-
-    Orchestrates ComplianceRetentionManager on a configurable timer
-    interval to automatically enforce retention rules.
-    """
-
-    def __init__(
-        self,
-        memory_db_path: str,
-        audit_db_path: str,
-        interval_seconds: int = DEFAULT_INTERVAL_SECONDS,
-    ):
-        self._memory_db_path = memory_db_path
-        self._audit_db_path = audit_db_path
-        self.interval_seconds = interval_seconds
-
-        self._manager = ComplianceRetentionManager(
-            memory_db_path, audit_db_path,
-        )
-
-        self._timer: Optional[threading.Timer] = None
-        self._running = False
-        self._lock = threading.Lock()
-
-    @property
-    def is_running(self) -> bool:
-        """Whether the scheduler is currently running."""
-        return self._running
-
-    def start(self) -> None:
-        """Start the background scheduler."""
-        with self._lock:
-            if self._running:
-                return
-            self._running = True
-            self._schedule_next()
-
-    def stop(self) -> None:
-        """Stop the background scheduler."""
-        with self._lock:
-            self._running = False
-            if self._timer is not None:
-                self._timer.cancel()
-                self._timer = None
-
-    def run_now(self) -> Dict[str, Any]:
-        """Execute a retention enforcement cycle immediately.
-
-        Returns:
-            Dict with timestamp, actions taken, and rules evaluated.
-        """
-        return self._execute_cycle()
-
-    def _schedule_next(self) -> None:
-        """Schedule the next enforcement cycle."""
-        self._timer = threading.Timer(self.interval_seconds, self._run_cycle)
-        self._timer.daemon = True
-        self._timer.start()
-
-    def _run_cycle(self) -> None:
-        """Run one enforcement cycle, then schedule the next."""
-        try:
-            self._execute_cycle()
-        except Exception:
-            pass  # Scheduler must not crash
-        finally:
-            with self._lock:
-                if self._running:
-                    self._schedule_next()
-
-    def _execute_cycle(self) -> Dict[str, Any]:
-        """Core retention enforcement logic.
-
-        1. Load all retention rules from audit.db
-        2. Scan every memory against each rule
-        3. Tombstone memories that exceed retention_days
-        4. Log actions to audit.db
-        """
-        rules = self._manager.list_rules()
-        actions: List[Dict[str, Any]] = []
-
-        # Scan all memories
-        memory_ids = self._get_all_memory_ids()
-
-        for mem_id in memory_ids:
-            mem = self._get_memory(mem_id)
-            if mem is None:
-                continue
-
-            # Already tombstoned -- skip
-            if mem.get("lifecycle_state") == "tombstoned":
-                continue
-
-            match = self._manager.evaluate_memory(mem_id)
-            if match is None:
-                continue
-
-            # Check if memory age exceeds the rule's retention_days
-            created_at = mem.get("created_at")
-            if created_at is None:
-                continue
-
-            age_days = self._age_in_days(created_at)
-            if age_days > match["retention_days"]:
-                action = match["action"]
-                if action == "tombstone":
-                    result = self._manager.execute_erasure_request(
-                        mem_id, match["framework"], "retention_scheduler",
-                    )
-                    actions.append({
-                        "memory_id": mem_id,
-                        "action": action,
-                        "rule_name": match["rule_name"],
-                        "framework": match["framework"],
-                        "age_days": age_days,
-                        "success": result.get("success", False),
-                    })
-
-        return {
-            "timestamp": datetime.now(timezone.utc).isoformat(),
-            "actions": actions,
-            "rules_evaluated": len(rules),
-        }
-
-    # ------------------------------------------------------------------
-    # Internal helpers
-    # ------------------------------------------------------------------
-
-    def _get_all_memory_ids(self) -> List[int]:
-        """Return all memory IDs from memory.db."""
-        conn = sqlite3.connect(self._memory_db_path)
-        try:
-            rows = conn.execute("SELECT id FROM memories").fetchall()
-            return [r[0] for r in rows]
-        finally:
-            conn.close()
-
-    def _get_memory(self, memory_id: int) -> Optional[Dict[str, Any]]:
-        """Fetch a single memory row as a dict."""
-        conn = sqlite3.connect(self._memory_db_path)
-        conn.row_factory = sqlite3.Row
-        try:
-            row = conn.execute(
-                "SELECT * FROM memories WHERE id = ?", (memory_id,),
-            ).fetchone()
-            return dict(row) if row else None
-        finally:
-            conn.close()
-
-    @staticmethod
-    def _age_in_days(created_at_str: str) -> float:
-        """Calculate age of a memory in days from its created_at."""
-        try:
-            created = datetime.fromisoformat(created_at_str)
-            now = datetime.now(created.tzinfo)
-            return (now - created).total_seconds() / 86400
-        except (ValueError, TypeError):
-            return 0.0

package/src/compliance/tests/__init__.py

@@ -1,4 +0,0 @@
-# SPDX-License-Identifier: MIT
-# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
-"""Tests for compliance engine.
-"""

package/src/compliance/tests/test_abac_enforcement.py

@@ -1,95 +0,0 @@
-# SPDX-License-Identifier: MIT
-# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
-"""Tests for ABAC enforcement in memory operations.
-"""
-import sqlite3
-import tempfile
-import os
-import sys
-import json
-import shutil
-from pathlib import Path
-
-sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent))
-
-
-class TestABACEnforcement:
-    def setup_method(self):
-        self.tmp_dir = tempfile.mkdtemp()
-        self.db_path = os.path.join(self.tmp_dir, "memory.db")
-        # Create store with test data
-        from memory_store_v2 import MemoryStoreV2
-        self.store = MemoryStoreV2(self.db_path)
-        self.store.add_memory(
-            content="public memory about Python",
-            tags=["python"],
-            importance=5,
-        )
-        self.store.add_memory(
-            content="private memory about secrets",
-            tags=["secrets"],
-            importance=8,
-        )
-        # Set access_level for private memory
-        conn = sqlite3.connect(self.db_path)
-        conn.execute("UPDATE memories SET access_level='private' WHERE id=2")
-        conn.commit()
-        conn.close()
-        self.store._rebuild_vectors()
-
-    def teardown_method(self):
-        shutil.rmtree(self.tmp_dir, ignore_errors=True)
-
-    def test_search_without_abac_works(self):
-        """Search without ABAC context works (backward compat)."""
-        results = self.store.search("Python", limit=5)
-        assert len(results) >= 1
-
-    def test_search_with_agent_context(self):
-        """Search with agent_context parameter works."""
-        results = self.store.search(
-            "memory", limit=10, agent_context={"agent_id": "user"}
-        )
-        assert isinstance(results, list)
-
-    def test_create_without_abac_works(self):
-        """Create without ABAC works (backward compat)."""
-        mem_id = self.store.add_memory(content="new memory", tags=["test"])
-        assert mem_id is not None
-
-    def test_abac_check_method_exists(self):
-        """MemoryStoreV2 has _check_abac method."""
-        assert hasattr(self.store, "_check_abac")
-
-    def test_check_abac_default_allows(self):
-        """Default ABAC check (no policy file) allows everything."""
-        result = self.store._check_abac(
-            subject={"agent_id": "user"},
-            resource={"access_level": "public"},
-            action="read",
-        )
-        assert result["allowed"] is True
-
-    def test_check_abac_with_policy(self):
-        """ABAC check with policy file respects deny rules."""
-        policy_path = os.path.join(self.tmp_dir, "abac_policies.json")
-        with open(policy_path, "w") as f:
-            json.dump(
-                [
-                    {
-                        "name": "deny-private",
-                        "effect": "deny",
-                        "subjects": {"agent_id": "*"},
-                        "resources": {"access_level": "private"},
-                        "actions": ["read"],
-                    }
-                ],
-                f,
-            )
-        result = self.store._check_abac(
-            subject={"agent_id": "bot"},
-            resource={"access_level": "private"},
-            action="read",
-            policy_path=policy_path,
-        )
-        assert result["allowed"] is False