superlocalmemory 2.7.6 → 2.8.0

package/src/compliance/retention_scheduler.py

@@ -0,0 +1,186 @@
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
+"""Background scheduler for periodic retention policy enforcement.
+
+Runs on a configurable interval (default: 24 hours) to:
+1. Load compliance retention rules from audit.db
+2. Scan all memories in memory.db against those rules
+3. Tombstone expired memories (age exceeds retention_days)
+4. Log every action to audit.db for tamper-evident compliance
+
+Uses daemon threading -- does not prevent process exit.
+"""
+import json
+import logging
+import sqlite3
+import threading
+from datetime import datetime, timezone
+from typing import Any, Dict, List, Optional
+
+from .retention_manager import ComplianceRetentionManager
+
+logger = logging.getLogger(__name__)
+
+# Default interval: 24 hours
+DEFAULT_INTERVAL_SECONDS = 86400
+
+
+class RetentionScheduler:
+    """Background scheduler for periodic retention policy enforcement.
+
+    Orchestrates ComplianceRetentionManager on a configurable timer
+    interval to automatically enforce retention rules.
+    """
+
+    def __init__(
+        self,
+        memory_db_path: str,
+        audit_db_path: str,
+        interval_seconds: int = DEFAULT_INTERVAL_SECONDS,
+    ):
+        self._memory_db_path = memory_db_path
+        self._audit_db_path = audit_db_path
+        self.interval_seconds = interval_seconds
+
+        self._manager = ComplianceRetentionManager(
+            memory_db_path, audit_db_path,
+        )
+
+        self._timer: Optional[threading.Timer] = None
+        self._running = False
+        self._lock = threading.Lock()
+
+    @property
+    def is_running(self) -> bool:
+        """Whether the scheduler is currently running."""
+        return self._running
+
+    def start(self) -> None:
+        """Start the background scheduler."""
+        with self._lock:
+            if self._running:
+                return
+            self._running = True
+            self._schedule_next()
+
+    def stop(self) -> None:
+        """Stop the background scheduler."""
+        with self._lock:
+            self._running = False
+            if self._timer is not None:
+                self._timer.cancel()
+                self._timer = None
+
+    def run_now(self) -> Dict[str, Any]:
+        """Execute a retention enforcement cycle immediately.
+
+        Returns:
+            Dict with timestamp, actions taken, and rules evaluated.
+        """
+        return self._execute_cycle()
+
+    def _schedule_next(self) -> None:
+        """Schedule the next enforcement cycle."""
+        self._timer = threading.Timer(self.interval_seconds, self._run_cycle)
+        self._timer.daemon = True
+        self._timer.start()
+
+    def _run_cycle(self) -> None:
+        """Run one enforcement cycle, then schedule the next."""
+        try:
+            self._execute_cycle()
+        except Exception:
+            pass  # Scheduler must not crash
+        finally:
+            with self._lock:
+                if self._running:
+                    self._schedule_next()
+
+    def _execute_cycle(self) -> Dict[str, Any]:
+        """Core retention enforcement logic.
+
+        1. Load all retention rules from audit.db
+        2. Scan every memory against each rule
+        3. Tombstone memories that exceed retention_days
+        4. Log actions to audit.db
+        """
+        rules = self._manager.list_rules()
+        actions: List[Dict[str, Any]] = []
+
+        # Scan all memories
+        memory_ids = self._get_all_memory_ids()
+
+        for mem_id in memory_ids:
+            mem = self._get_memory(mem_id)
+            if mem is None:
+                continue
+
+            # Already tombstoned -- skip
+            if mem.get("lifecycle_state") == "tombstoned":
+                continue
+
+            match = self._manager.evaluate_memory(mem_id)
+            if match is None:
+                continue
+
+            # Check if memory age exceeds the rule's retention_days
+            created_at = mem.get("created_at")
+            if created_at is None:
+                continue
+
+            age_days = self._age_in_days(created_at)
+            if age_days > match["retention_days"]:
+                action = match["action"]
+                if action == "tombstone":
+                    result = self._manager.execute_erasure_request(
+                        mem_id, match["framework"], "retention_scheduler",
+                    )
+                    actions.append({
+                        "memory_id": mem_id,
+                        "action": action,
+                        "rule_name": match["rule_name"],
+                        "framework": match["framework"],
+                        "age_days": age_days,
+                        "success": result.get("success", False),
+                    })
+
+        return {
+            "timestamp": datetime.now(timezone.utc).isoformat(),
+            "actions": actions,
+            "rules_evaluated": len(rules),
+        }
+
+    # ------------------------------------------------------------------
+    # Internal helpers
+    # ------------------------------------------------------------------
+
+    def _get_all_memory_ids(self) -> List[int]:
+        """Return all memory IDs from memory.db."""
+        conn = sqlite3.connect(self._memory_db_path)
+        try:
+            rows = conn.execute("SELECT id FROM memories").fetchall()
+            return [r[0] for r in rows]
+        finally:
+            conn.close()
+
+    def _get_memory(self, memory_id: int) -> Optional[Dict[str, Any]]:
+        """Fetch a single memory row as a dict."""
+        conn = sqlite3.connect(self._memory_db_path)
+        conn.row_factory = sqlite3.Row
+        try:
+            row = conn.execute(
+                "SELECT * FROM memories WHERE id = ?", (memory_id,),
+            ).fetchone()
+            return dict(row) if row else None
+        finally:
+            conn.close()
+
+    @staticmethod
+    def _age_in_days(created_at_str: str) -> float:
+        """Calculate age of a memory in days from its created_at."""
+        try:
+            created = datetime.fromisoformat(created_at_str)
+            now = datetime.now(created.tzinfo)
+            return (now - created).total_seconds() / 86400
+        except (ValueError, TypeError):
+            return 0.0

package/src/compliance/tests/__init__.py

@@ -0,0 +1,4 @@
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
+"""Tests for compliance engine.
+"""

package/src/compliance/tests/test_abac_enforcement.py

@@ -0,0 +1,95 @@
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
+"""Tests for ABAC enforcement in memory operations.
+"""
+import sqlite3
+import tempfile
+import os
+import sys
+import json
+import shutil
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent))
+
+
+class TestABACEnforcement:
+    def setup_method(self):
+        self.tmp_dir = tempfile.mkdtemp()
+        self.db_path = os.path.join(self.tmp_dir, "memory.db")
+        # Create store with test data
+        from memory_store_v2 import MemoryStoreV2
+        self.store = MemoryStoreV2(self.db_path)
+        self.store.add_memory(
+            content="public memory about Python",
+            tags=["python"],
+            importance=5,
+        )
+        self.store.add_memory(
+            content="private memory about secrets",
+            tags=["secrets"],
+            importance=8,
+        )
+        # Set access_level for private memory
+        conn = sqlite3.connect(self.db_path)
+        conn.execute("UPDATE memories SET access_level='private' WHERE id=2")
+        conn.commit()
+        conn.close()
+        self.store._rebuild_vectors()
+
+    def teardown_method(self):
+        shutil.rmtree(self.tmp_dir, ignore_errors=True)
+
+    def test_search_without_abac_works(self):
+        """Search without ABAC context works (backward compat)."""
+        results = self.store.search("Python", limit=5)
+        assert len(results) >= 1
+
+    def test_search_with_agent_context(self):
+        """Search with agent_context parameter works."""
+        results = self.store.search(
+            "memory", limit=10, agent_context={"agent_id": "user"}
+        )
+        assert isinstance(results, list)
+
+    def test_create_without_abac_works(self):
+        """Create without ABAC works (backward compat)."""
+        mem_id = self.store.add_memory(content="new memory", tags=["test"])
+        assert mem_id is not None
+
+    def test_abac_check_method_exists(self):
+        """MemoryStoreV2 has _check_abac method."""
+        assert hasattr(self.store, "_check_abac")
+
+    def test_check_abac_default_allows(self):
+        """Default ABAC check (no policy file) allows everything."""
+        result = self.store._check_abac(
+            subject={"agent_id": "user"},
+            resource={"access_level": "public"},
+            action="read",
+        )
+        assert result["allowed"] is True
+
+    def test_check_abac_with_policy(self):
+        """ABAC check with policy file respects deny rules."""
+        policy_path = os.path.join(self.tmp_dir, "abac_policies.json")
+        with open(policy_path, "w") as f:
+            json.dump(
+                [
+                    {
+                        "name": "deny-private",
+                        "effect": "deny",
+                        "subjects": {"agent_id": "*"},
+                        "resources": {"access_level": "private"},
+                        "actions": ["read"],
+                    }
+                ],
+                f,
+            )
+        result = self.store._check_abac(
+            subject={"agent_id": "bot"},
+            resource={"access_level": "private"},
+            action="read",
+            policy_path=policy_path,
+        )
+        assert result["allowed"] is False

package/src/compliance/tests/test_abac_engine.py

@@ -0,0 +1,124 @@
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
+"""Tests for ABAC policy engine.
+"""
+import tempfile
+import os
+import sys
+import json
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent))
+
+
+class TestABACEngine:
+    def setup_method(self):
+        self.tmp_dir = tempfile.mkdtemp()
+        self.policy_path = os.path.join(self.tmp_dir, "abac_policies.json")
+
+    def teardown_method(self):
+        import shutil
+        shutil.rmtree(self.tmp_dir, ignore_errors=True)
+
+    def _write_policies(self, policies):
+        with open(self.policy_path, "w") as f:
+            json.dump(policies, f)
+
+    def test_creation_no_policy_file(self):
+        """Engine works with no policy file — allow all."""
+        from compliance.abac_engine import ABACEngine
+        engine = ABACEngine(config_path="/nonexistent/path.json")
+        assert engine is not None
+
+    def test_missing_policy_allows_all(self):
+        """No policy file → all access allowed (backward compat)."""
+        from compliance.abac_engine import ABACEngine
+        engine = ABACEngine(config_path="/nonexistent/path.json")
+        result = engine.evaluate(subject={"agent_id": "user"}, resource={"access_level": "public"}, action="read")
+        assert result["allowed"] is True
+
+    def test_load_policies_from_json(self):
+        """Can load policies from JSON file."""
+        from compliance.abac_engine import ABACEngine
+        self._write_policies([
+            {"name": "deny-private", "effect": "deny", "subjects": {"agent_id": "*"}, "resources": {"access_level": "private"}, "actions": ["read"]}
+        ])
+        engine = ABACEngine(config_path=self.policy_path)
+        assert len(engine.policies) == 1
+
+    def test_deny_policy_blocks_access(self):
+        """Deny policy prevents access to matching resources."""
+        from compliance.abac_engine import ABACEngine
+        self._write_policies([
+            {"name": "deny-private", "effect": "deny", "subjects": {"agent_id": "*"}, "resources": {"access_level": "private"}, "actions": ["read"]}
+        ])
+        engine = ABACEngine(config_path=self.policy_path)
+        result = engine.evaluate(subject={"agent_id": "agent_a"}, resource={"access_level": "private"}, action="read")
+        assert result["allowed"] is False
+        assert result["policy_name"] == "deny-private"
+
+    def test_allow_policy_grants_access(self):
+        """Allow policy explicitly permits access."""
+        from compliance.abac_engine import ABACEngine
+        self._write_policies([
+            {"name": "allow-admin", "effect": "allow", "subjects": {"agent_id": "admin"}, "resources": {"access_level": "*"}, "actions": ["read", "write", "delete"]}
+        ])
+        engine = ABACEngine(config_path=self.policy_path)
+        result = engine.evaluate(subject={"agent_id": "admin"}, resource={"access_level": "private"}, action="write")
+        assert result["allowed"] is True
+
+    def test_subject_matching_specific_agent(self):
+        """Policy matches specific agent_id."""
+        from compliance.abac_engine import ABACEngine
+        self._write_policies([
+            {"name": "deny-untrusted", "effect": "deny", "subjects": {"agent_id": "untrusted_bot"}, "resources": {"access_level": "*"}, "actions": ["read"]}
+        ])
+        engine = ABACEngine(config_path=self.policy_path)
+        # untrusted_bot denied
+        r1 = engine.evaluate(subject={"agent_id": "untrusted_bot"}, resource={"access_level": "public"}, action="read")
+        assert r1["allowed"] is False
+        # trusted_agent allowed (no matching deny policy)
+        r2 = engine.evaluate(subject={"agent_id": "trusted_agent"}, resource={"access_level": "public"}, action="read")
+        assert r2["allowed"] is True
+
+    def test_resource_matching_by_project(self):
+        """Policy matches by project name."""
+        from compliance.abac_engine import ABACEngine
+        self._write_policies([
+            {"name": "deny-secret-project", "effect": "deny", "subjects": {"agent_id": "*"}, "resources": {"project": "secret_project"}, "actions": ["read"]}
+        ])
+        engine = ABACEngine(config_path=self.policy_path)
+        r1 = engine.evaluate(subject={"agent_id": "user"}, resource={"project": "secret_project"}, action="read")
+        assert r1["allowed"] is False
+        r2 = engine.evaluate(subject={"agent_id": "user"}, resource={"project": "public_project"}, action="read")
+        assert r2["allowed"] is True
+
+    def test_action_matching(self):
+        """Policy only applies to specified actions."""
+        from compliance.abac_engine import ABACEngine
+        self._write_policies([
+            {"name": "deny-delete", "effect": "deny", "subjects": {"agent_id": "*"}, "resources": {"access_level": "*"}, "actions": ["delete"]}
+        ])
+        engine = ABACEngine(config_path=self.policy_path)
+        r1 = engine.evaluate(subject={"agent_id": "user"}, resource={"access_level": "public"}, action="delete")
+        assert r1["allowed"] is False
+        r2 = engine.evaluate(subject={"agent_id": "user"}, resource={"access_level": "public"}, action="read")
+        assert r2["allowed"] is True
+
+    def test_deny_takes_precedence(self):
+        """When both allow and deny match, deny wins."""
+        from compliance.abac_engine import ABACEngine
+        self._write_policies([
+            {"name": "allow-all", "effect": "allow", "subjects": {"agent_id": "*"}, "resources": {"access_level": "*"}, "actions": ["read"]},
+            {"name": "deny-private", "effect": "deny", "subjects": {"agent_id": "*"}, "resources": {"access_level": "private"}, "actions": ["read"]}
+        ])
+        engine = ABACEngine(config_path=self.policy_path)
+        result = engine.evaluate(subject={"agent_id": "user"}, resource={"access_level": "private"}, action="read")
+        assert result["allowed"] is False
+
+    def test_evaluate_returns_reason(self):
+        """Evaluation result includes reason."""
+        from compliance.abac_engine import ABACEngine
+        engine = ABACEngine(config_path="/nonexistent/path.json")
+        result = engine.evaluate(subject={"agent_id": "user"}, resource={}, action="read")
+        assert "reason" in result

package/src/compliance/tests/test_abac_mcp_integration.py

@@ -0,0 +1,118 @@
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
+"""Tests for ABAC enforcement via MCP tool integration.
+"""
+import tempfile
+import os
+import sys
+import json
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent))
+
+
+class TestABACMCPIntegration:
+    def setup_method(self):
+        self.tmp_dir = tempfile.mkdtemp()
+        self.db_path = os.path.join(self.tmp_dir, "memory.db")
+
+    def teardown_method(self):
+        import shutil
+
+        shutil.rmtree(self.tmp_dir, ignore_errors=True)
+
+    def test_middleware_creation(self):
+        from compliance.abac_middleware import ABACMiddleware
+
+        mw = ABACMiddleware(self.db_path)
+        assert mw is not None
+
+    def test_check_read_access_default_allow(self):
+        """Default (no policies) allows all reads."""
+        from compliance.abac_middleware import ABACMiddleware
+
+        mw = ABACMiddleware(self.db_path)
+        result = mw.check_access(
+            agent_id="any_agent",
+            action="read",
+            resource={"access_level": "public"},
+        )
+        assert result["allowed"] is True
+
+    def test_check_write_access_default_allow(self):
+        from compliance.abac_middleware import ABACMiddleware
+
+        mw = ABACMiddleware(self.db_path)
+        result = mw.check_access(
+            agent_id="any_agent", action="write", resource={}
+        )
+        assert result["allowed"] is True
+
+    def test_check_access_with_deny_policy(self):
+        """Deny policy blocks access when enforced."""
+        from compliance.abac_middleware import ABACMiddleware
+
+        policy_path = os.path.join(self.tmp_dir, "abac_policies.json")
+        with open(policy_path, "w") as f:
+            json.dump(
+                [
+                    {
+                        "name": "deny-bots",
+                        "effect": "deny",
+                        "subjects": {"agent_id": "untrusted_bot"},
+                        "resources": {"access_level": "*"},
+                        "actions": ["read", "write"],
+                    }
+                ],
+                f,
+            )
+        mw = ABACMiddleware(self.db_path, policy_path=policy_path)
+        result = mw.check_access(
+            agent_id="untrusted_bot",
+            action="read",
+            resource={"access_level": "public"},
+        )
+        assert result["allowed"] is False
+
+    def test_denied_access_logged(self):
+        """Denied access is recorded for audit trail."""
+        from compliance.abac_middleware import ABACMiddleware
+
+        policy_path = os.path.join(self.tmp_dir, "abac_policies.json")
+        with open(policy_path, "w") as f:
+            json.dump(
+                [
+                    {
+                        "name": "deny-all-write",
+                        "effect": "deny",
+                        "subjects": {"agent_id": "*"},
+                        "resources": {"access_level": "*"},
+                        "actions": ["write"],
+                    }
+                ],
+                f,
+            )
+        mw = ABACMiddleware(self.db_path, policy_path=policy_path)
+        mw.check_access(agent_id="user", action="write", resource={})
+        assert mw.denied_count >= 1
+
+    def test_build_agent_context(self):
+        """build_agent_context creates proper context dict for store."""
+        from compliance.abac_middleware import ABACMiddleware
+
+        mw = ABACMiddleware(self.db_path)
+        ctx = mw.build_agent_context(agent_id="claude_agent", protocol="mcp")
+        assert ctx["agent_id"] == "claude_agent"
+        assert ctx["protocol"] == "mcp"
+
+    def test_graceful_when_compliance_unavailable(self):
+        """Middleware works even if ABACEngine import fails."""
+        from compliance.abac_middleware import ABACMiddleware
+
+        mw = ABACMiddleware(
+            self.db_path, policy_path="/nonexistent/path.json"
+        )
+        result = mw.check_access(
+            agent_id="user", action="read", resource={}
+        )
+        assert result["allowed"] is True

package/src/compliance/tests/test_audit_db.py

@@ -0,0 +1,123 @@
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
+"""Tests for audit database with hash chain tamper detection.
+"""
+import sqlite3
+import tempfile
+import os
+import sys
+import hashlib
+import json
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent))
+
+
+class TestAuditDB:
+    def setup_method(self):
+        self.tmp_dir = tempfile.mkdtemp()
+        self.db_path = os.path.join(self.tmp_dir, "audit.db")
+
+    def teardown_method(self):
+        import shutil
+        shutil.rmtree(self.tmp_dir, ignore_errors=True)
+
+    def test_creation(self):
+        from compliance.audit_db import AuditDB
+        db = AuditDB(self.db_path)
+        assert db is not None
+
+    def test_schema_created(self):
+        from compliance.audit_db import AuditDB
+        db = AuditDB(self.db_path)
+        conn = sqlite3.connect(self.db_path)
+        tables = {r[0] for r in conn.execute("SELECT name FROM sqlite_master WHERE type='table'").fetchall()}
+        conn.close()
+        assert "audit_events" in tables
+
+    def test_log_event(self):
+        from compliance.audit_db import AuditDB
+        db = AuditDB(self.db_path)
+        eid = db.log_event(event_type="memory.created", actor="user", resource_id=1, details={"action": "create"})
+        assert isinstance(eid, int)
+        assert eid > 0
+
+    def test_hash_chain_first_entry(self):
+        """First entry's prev_hash should be a known genesis value."""
+        from compliance.audit_db import AuditDB
+        db = AuditDB(self.db_path)
+        db.log_event("memory.created", actor="user", resource_id=1)
+        conn = sqlite3.connect(self.db_path)
+        row = conn.execute("SELECT prev_hash, entry_hash FROM audit_events WHERE id=1").fetchone()
+        conn.close()
+        assert row[0] == "genesis"
+        assert row[1] is not None and len(row[1]) == 64  # SHA-256 hex
+
+    def test_hash_chain_links(self):
+        """Each entry's prev_hash should equal the previous entry's entry_hash."""
+        from compliance.audit_db import AuditDB
+        db = AuditDB(self.db_path)
+        db.log_event("memory.created", actor="user", resource_id=1)
+        db.log_event("memory.recalled", actor="agent_a", resource_id=2)
+        db.log_event("memory.deleted", actor="user", resource_id=1)
+        conn = sqlite3.connect(self.db_path)
+        rows = conn.execute("SELECT id, prev_hash, entry_hash FROM audit_events ORDER BY id").fetchall()
+        conn.close()
+        assert rows[1][1] == rows[0][2]  # Entry 2's prev = Entry 1's hash
+        assert rows[2][1] == rows[1][2]  # Entry 3's prev = Entry 2's hash
+
+    def test_verify_chain_valid(self):
+        """verify_chain returns True for untampered chain."""
+        from compliance.audit_db import AuditDB
+        db = AuditDB(self.db_path)
+        db.log_event("memory.created", actor="user", resource_id=1)
+        db.log_event("memory.recalled", actor="agent_a", resource_id=1)
+        result = db.verify_chain()
+        assert result["valid"] is True
+        assert result["entries_checked"] == 2
+
+    def test_verify_chain_detects_tampering(self):
+        """verify_chain returns False if an entry was modified."""
+        from compliance.audit_db import AuditDB
+        db = AuditDB(self.db_path)
+        db.log_event("memory.created", actor="user", resource_id=1)
+        db.log_event("memory.recalled", actor="agent_a", resource_id=1)
+        # Tamper with the first entry
+        conn = sqlite3.connect(self.db_path)
+        conn.execute("UPDATE audit_events SET actor='hacker' WHERE id=1")
+        conn.commit()
+        conn.close()
+        result = db.verify_chain()
+        assert result["valid"] is False
+
+    def test_query_by_type(self):
+        from compliance.audit_db import AuditDB
+        db = AuditDB(self.db_path)
+        db.log_event("memory.created", actor="user", resource_id=1)
+        db.log_event("memory.recalled", actor="user", resource_id=1)
+        db.log_event("memory.created", actor="user", resource_id=2)
+        results = db.query_events(event_type="memory.created")
+        assert len(results) == 2
+
+    def test_query_by_actor(self):
+        from compliance.audit_db import AuditDB
+        db = AuditDB(self.db_path)
+        db.log_event("memory.created", actor="user", resource_id=1)
+        db.log_event("memory.recalled", actor="agent_a", resource_id=1)
+        results = db.query_events(actor="agent_a")
+        assert len(results) == 1
+
+    def test_query_by_time_range(self):
+        from compliance.audit_db import AuditDB
+        db = AuditDB(self.db_path)
+        db.log_event("memory.created", actor="user", resource_id=1)
+        results = db.query_events(limit=10)
+        assert len(results) >= 1
+        assert "created_at" in results[0]
+
+    def test_empty_chain_is_valid(self):
+        from compliance.audit_db import AuditDB
+        db = AuditDB(self.db_path)
+        result = db.verify_chain()
+        assert result["valid"] is True
+        assert result["entries_checked"] == 0