superlocalmemory 2.8.6 → 3.0.0

package/src/superlocalmemory/compliance/gdpr.py

@@ -0,0 +1,294 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under the MIT License - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""SuperLocalMemory V3 — GDPR Compliance.
+
+Implements GDPR rights: right to access, right to erasure (forget),
+right to data portability (export), and audit trail.
+Profile-scoped. All operations logged to compliance_audit.
+
+Part of Qualixar | Author: Varun Pratap Bhardwaj
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from datetime import UTC, datetime
+
+logger = logging.getLogger(__name__)
+
+
+class GDPRCompliance:
+    """GDPR compliance operations for memory data.
+
+    Supports:
+    - Right to Access (Art. 15): Export all data for a profile
+    - Right to Erasure (Art. 17): Delete all data for a profile/entity
+    - Right to Portability (Art. 20): Export in machine-readable format
+    - Audit Trail: Log all data operations
+    """
+
+    def __init__(self, db) -> None:
+        self._db = db
+
+    # -- Right to Access (Art. 15) -----------------------------------------
+
+    def export_profile_data(self, profile_id: str) -> dict:
+        """Export ALL data for a profile in machine-readable format.
+
+        Returns a dict containing all memories, facts, entities,
+        edges, trust scores, feedback, and behavioral patterns.
+        """
+        self._audit("export", "profile", profile_id, "Full data export")
+
+        data: dict = {"profile_id": profile_id, "exported_at": _now()}
+
+        # Memories
+        rows = self._db.execute(
+            "SELECT * FROM memories WHERE profile_id = ?", (profile_id,)
+        )
+        data["memories"] = [dict(r) for r in rows]
+
+        # Facts
+        rows = self._db.execute(
+            "SELECT * FROM atomic_facts WHERE profile_id = ?", (profile_id,)
+        )
+        data["facts"] = [dict(r) for r in rows]
+
+        # Entities
+        rows = self._db.execute(
+            "SELECT * FROM canonical_entities WHERE profile_id = ?", (profile_id,)
+        )
+        data["entities"] = [dict(r) for r in rows]
+
+        # Graph edges
+        rows = self._db.execute(
+            "SELECT * FROM graph_edges WHERE profile_id = ?", (profile_id,)
+        )
+        data["edges"] = [dict(r) for r in rows]
+
+        # Trust scores
+        rows = self._db.execute(
+            "SELECT * FROM trust_scores WHERE profile_id = ?", (profile_id,)
+        )
+        data["trust_scores"] = [dict(r) for r in rows]
+
+        # Feedback
+        rows = self._db.execute(
+            "SELECT * FROM feedback_records WHERE profile_id = ?", (profile_id,)
+        )
+        data["feedback"] = [dict(r) for r in rows]
+
+        # Entity profiles
+        rows = self._db.execute(
+            "SELECT * FROM entity_profiles WHERE profile_id = ?", (profile_id,)
+        )
+        data["entity_profiles"] = [dict(r) for r in rows]
+
+        # Memory scenes
+        rows = self._db.execute(
+            "SELECT * FROM memory_scenes WHERE profile_id = ?", (profile_id,)
+        )
+        data["scenes"] = [dict(r) for r in rows]
+
+        # Temporal events
+        rows = self._db.execute(
+            "SELECT * FROM temporal_events WHERE profile_id = ?", (profile_id,)
+        )
+        data["temporal_events"] = [dict(r) for r in rows]
+
+        # Consolidation log
+        rows = self._db.execute(
+            "SELECT * FROM consolidation_log WHERE profile_id = ?", (profile_id,)
+        )
+        data["consolidation_log"] = [dict(r) for r in rows]
+
+        # Behavioral patterns
+        rows = self._db.execute(
+            "SELECT * FROM behavioral_patterns WHERE profile_id = ?", (profile_id,)
+        )
+        data["behavioral_patterns"] = [dict(r) for r in rows]
+
+        # Action outcomes
+        rows = self._db.execute(
+            "SELECT * FROM action_outcomes WHERE profile_id = ?", (profile_id,)
+        )
+        data["action_outcomes"] = [dict(r) for r in rows]
+
+        # Compliance audit trail
+        rows = self._db.execute(
+            "SELECT * FROM compliance_audit WHERE profile_id = ?", (profile_id,)
+        )
+        data["compliance_audit"] = [dict(r) for r in rows]
+
+        # Provenance (data lineage — EU AI Act Art. 10)
+        rows = self._db.execute(
+            "SELECT * FROM provenance WHERE profile_id = ?", (profile_id,)
+        )
+        data["provenance"] = [dict(r) for r in rows]
+
+        # Entity aliases (indirect PII via entity relationships)
+        rows = self._db.execute(
+            "SELECT ea.* FROM entity_aliases ea "
+            "JOIN canonical_entities ce ON ea.entity_id = ce.entity_id "
+            "WHERE ce.profile_id = ?", (profile_id,)
+        )
+        data["entity_aliases"] = [dict(r) for r in rows]
+
+        # Profile record itself
+        rows = self._db.execute(
+            "SELECT * FROM profiles WHERE profile_id = ?", (profile_id,)
+        )
+        data["profile_record"] = [dict(r) for r in rows]
+
+        data["total_items"] = sum(
+            len(v) for v in data.values() if isinstance(v, list)
+        )
+
+        logger.info("Exported %d items for profile '%s'", data["total_items"], profile_id)
+        return data
+
+    # -- Right to Erasure (Art. 17) ----------------------------------------
+
+    def forget_profile(self, profile_id: str) -> dict:
+        """Delete ALL data for a profile (right to be forgotten).
+
+        CASCADE deletes handle most cleanup via foreign keys.
+        Returns counts of deleted items.
+        """
+        if profile_id == "default":
+            raise ValueError("Cannot delete the default profile via GDPR erasure. "
+                             "Use profile deletion instead.")
+
+        self._audit("delete", "profile", profile_id, "GDPR erasure request")
+
+        counts: dict[str, int] = {}
+        tables = [
+            "compliance_audit", "action_outcomes", "behavioral_patterns",
+            "feedback_records", "trust_scores", "provenance",
+            "consolidation_log", "graph_edges", "temporal_events",
+            "memory_scenes", "entity_profiles", "bm25_tokens",
+            "atomic_facts", "memories", "canonical_entities",
+        ]
+        for table in tables:
+            rows = self._db.execute(
+                f"SELECT COUNT(*) AS c FROM {table} WHERE profile_id = ?",
+                (profile_id,),
+            )
+            counts[table] = int(dict(rows[0])["c"]) if rows else 0
+            self._db.execute(
+                f"DELETE FROM {table} WHERE profile_id = ?", (profile_id,)
+            )
+
+        # Delete entity aliases (orphan PII via entity relationships)
+        self._db.execute(
+            "DELETE FROM entity_aliases WHERE entity_id IN "
+            "(SELECT entity_id FROM canonical_entities WHERE profile_id = ?)",
+            (profile_id,),
+        )
+
+        # Delete profile itself
+        self._db.execute(
+            "DELETE FROM profiles WHERE profile_id = ?", (profile_id,)
+        )
+        counts["profiles"] = 1
+
+        # Erase learning database (separate DB file)
+        try:
+            from superlocalmemory.learning.database import LearningDatabase
+            from superlocalmemory.core.config import DEFAULT_BASE_DIR
+            learning_db = LearningDatabase(DEFAULT_BASE_DIR / "learning.db")
+            learning_db.reset(profile_id)
+            counts["learning_db"] = 1
+        except Exception:
+            pass
+
+        # VACUUM to remove deleted data from physical file
+        try:
+            self._db.execute("VACUUM")
+        except Exception:
+            pass
+
+        logger.info("GDPR erasure for '%s': %s", profile_id, counts)
+        return counts
+
+    def forget_entity(self, entity_name: str, profile_id: str) -> dict:
+        """Delete all data related to a specific entity.
+
+        Removes facts mentioning the entity, edges, temporal events,
+        and the entity itself. For targeted erasure requests.
+        """
+        self._audit("delete", "entity", entity_name,
+                     f"GDPR entity erasure in profile {profile_id}",
+                     profile_id=profile_id)
+
+        entity = self._db.get_entity_by_name(entity_name, profile_id)
+        if entity is None:
+            return {"deleted": 0, "entity": entity_name, "found": False}
+
+        eid = entity.entity_id
+        counts: dict[str, int] = {}
+
+        # Delete facts mentioning this entity
+        rows = self._db.execute(
+            "SELECT fact_id FROM atomic_facts WHERE profile_id = ? "
+            "AND canonical_entities_json LIKE ?",
+            (profile_id, f'%"{eid}"%'),
+        )
+        fact_ids = [dict(r)["fact_id"] for r in rows]
+        for fid in fact_ids:
+            self._db.delete_fact(fid)
+        counts["facts"] = len(fact_ids)
+
+        # Delete temporal events
+        self._db.execute(
+            "DELETE FROM temporal_events WHERE entity_id = ? AND profile_id = ?",
+            (eid, profile_id),
+        )
+
+        # Delete entity profile
+        self._db.execute(
+            "DELETE FROM entity_profiles WHERE entity_id = ? AND profile_id = ?",
+            (eid, profile_id),
+        )
+
+        # Delete aliases + entity
+        self._db.execute("DELETE FROM entity_aliases WHERE entity_id = ?", (eid,))
+        self._db.execute("DELETE FROM canonical_entities WHERE entity_id = ?", (eid,))
+        counts["entity"] = 1
+
+        logger.info("Entity erasure '%s' in '%s': %s", entity_name, profile_id, counts)
+        return counts
+
+    # -- Audit Trail -------------------------------------------------------
+
+    def get_audit_trail(
+        self, profile_id: str, limit: int = 100
+    ) -> list[dict]:
+        """Get compliance audit trail for a profile."""
+        rows = self._db.execute(
+            "SELECT * FROM compliance_audit WHERE profile_id = ? "
+            "ORDER BY timestamp DESC LIMIT ?",
+            (profile_id, limit),
+        )
+        return [dict(r) for r in rows]
+
+    def _audit(
+        self, action: str, target_type: str, target_id: str, details: str,
+        profile_id: str | None = None,
+    ) -> None:
+        """Log a compliance action."""
+        from superlocalmemory.storage.models import _new_id
+        pid = profile_id if profile_id is not None else target_id
+        self._db.execute(
+            "INSERT INTO compliance_audit "
+            "(audit_id, profile_id, action, target_type, target_id, details, timestamp) "
+            "VALUES (?,?,?,?,?,?,?)",
+            (_new_id(), pid, action, target_type, target_id, details, _now()),
+        )
+
+
+def _now() -> str:
+    return datetime.now(UTC).isoformat()

package/src/superlocalmemory/compliance/lifecycle.py

@@ -0,0 +1,158 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under the MIT License - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""SuperLocalMemory V3 — Memory Lifecycle Management.
+
+Implements Active → Warm → Cold → Archived state machine.
+Coupled with Langevin dynamics: positions naturally create lifecycle states.
+
+Ported from V2.8 with Langevin coupling (Innovation's unique feature).
+
+Part of Qualixar | Author: Varun Pratap Bhardwaj
+"""
+
+from __future__ import annotations
+
+import logging
+from datetime import UTC, datetime, timedelta
+
+from superlocalmemory.storage.models import AtomicFact, MemoryLifecycle
+
+logger = logging.getLogger(__name__)
+
+# Lifecycle transition thresholds (days since last access)
+_ACTIVE_MAX_DAYS = 7        # Active for 7 days after last access
+_WARM_MAX_DAYS = 30         # Warm for 30 days
+_COLD_MAX_DAYS = 90         # Cold for 90 days, then archived
+
+# Langevin weight thresholds (if Langevin is available)
+_ACTIVE_WEIGHT_MIN = 0.7
+_WARM_WEIGHT_MIN = 0.4
+_COLD_WEIGHT_MIN = 0.1
+
+
+class LifecycleManager:
+    """Manage memory lifecycle states.
+
+    Two complementary strategies:
+    1. Time-based: days since last access → state transition
+    2. Langevin-based: position on Poincaré ball → lifecycle weight → state
+
+    When Langevin is available, it takes precedence (more nuanced).
+    Time-based is the fallback for Mode A (no dynamics).
+    """
+
+    def __init__(self, db, langevin=None) -> None:
+        self._db = db
+        self._langevin = langevin
+
+    def get_lifecycle_state(self, fact: AtomicFact) -> MemoryLifecycle:
+        """Determine current lifecycle state for a fact."""
+        # Strategy 1: Langevin-based (if available and position exists)
+        if self._langevin is not None and fact.langevin_position:
+            weight = self._langevin.compute_lifecycle_weight(fact.langevin_position)
+            return self._langevin.get_lifecycle_state(weight)
+
+        # Strategy 2: Time-based fallback
+        return self._time_based_state(fact)
+
+    def update_lifecycle(self, fact_id: str, profile_id: str) -> MemoryLifecycle:
+        """Recompute and persist lifecycle state for a fact."""
+        rows = self._db.execute(
+            "SELECT lifecycle, access_count, created_at FROM atomic_facts "
+            "WHERE fact_id = ? AND profile_id = ?",
+            (fact_id, profile_id),
+        )
+        if not rows:
+            return MemoryLifecycle.ACTIVE
+
+        d = dict(rows[0])
+        days = self._days_since(d.get("created_at", ""))
+        access = d.get("access_count", 0)
+
+        if access > 10 or days < _ACTIVE_MAX_DAYS:
+            state = MemoryLifecycle.ACTIVE
+        elif days < _WARM_MAX_DAYS:
+            state = MemoryLifecycle.WARM
+        elif days < _COLD_MAX_DAYS:
+            state = MemoryLifecycle.COLD
+        else:
+            state = MemoryLifecycle.ARCHIVED
+
+        current = d.get("lifecycle", "active")
+        if current != state.value:
+            self._db.update_fact(fact_id, {"lifecycle": state})
+            logger.debug("Fact %s: %s → %s", fact_id, current, state.value)
+
+        return state
+
+    def run_maintenance(self, profile_id: str) -> dict[str, int]:
+        """Run lifecycle maintenance on all facts in a profile.
+
+        If Langevin is available, evolve positions and update states.
+        Otherwise, use time-based transitions.
+        """
+        facts = self._db.get_all_facts(profile_id)
+        counts: dict[str, int] = {s.value: 0 for s in MemoryLifecycle}
+        transitions = 0
+
+        for fact in facts:
+            old_state = fact.lifecycle
+
+            if self._langevin is not None and fact.langevin_position:
+                # Langevin step
+                age = self._days_since(fact.created_at)
+                new_pos, weight = self._langevin.step(
+                    fact.langevin_position, fact.access_count, age, fact.importance
+                )
+                new_state = self._langevin.get_lifecycle_state(weight)
+                # Persist position + state
+                self._db.update_fact(fact.fact_id, {
+                    "langevin_position": new_pos,
+                    "lifecycle": new_state,
+                })
+            else:
+                new_state = self._time_based_state(fact)
+                if new_state != old_state:
+                    self._db.update_fact(fact.fact_id, {"lifecycle": new_state})
+
+            counts[new_state.value] = counts.get(new_state.value, 0) + 1
+            if new_state != old_state:
+                transitions += 1
+
+        counts["transitions"] = transitions
+        logger.info("Lifecycle maintenance for '%s': %s", profile_id, counts)
+        return counts
+
+    def get_archived_facts(self, profile_id: str) -> list[AtomicFact]:
+        """Get all archived facts (candidates for deletion/export)."""
+        rows = self._db.execute(
+            "SELECT * FROM atomic_facts WHERE profile_id = ? AND lifecycle = 'archived'",
+            (profile_id,),
+        )
+        return [self._db._row_to_fact(r) for r in rows]
+
+    # -- Internal ----------------------------------------------------------
+
+    def _time_based_state(self, fact: AtomicFact) -> MemoryLifecycle:
+        """Determine lifecycle state from time since creation + access count."""
+        days = self._days_since(fact.created_at)
+        if fact.access_count > 10 or days < _ACTIVE_MAX_DAYS:
+            return MemoryLifecycle.ACTIVE
+        if days < _WARM_MAX_DAYS:
+            return MemoryLifecycle.WARM
+        if days < _COLD_MAX_DAYS:
+            return MemoryLifecycle.COLD
+        return MemoryLifecycle.ARCHIVED
+
+    @staticmethod
+    def _days_since(iso_date: str) -> float:
+        """Days since an ISO date string."""
+        if not iso_date:
+            return 0.0
+        try:
+            dt = datetime.fromisoformat(iso_date)
+            return (datetime.now(UTC) - dt).total_seconds() / 86400.0
+        except (ValueError, TypeError):
+            return 0.0

package/src/superlocalmemory/compliance/retention.py

@@ -0,0 +1,232 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under the MIT License - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""Named retention rules engine for compliance (GDPR, HIPAA, custom).
+
+Rules are bound to profiles. Each rule specifies a retention period
+in days. The engine can identify expired facts and enforce deletion.
+
+Retention rules are stored in a dedicated SQLite table and operate
+independently of the main memory lifecycle system.
+"""
+
+from __future__ import annotations
+
+import logging
+import sqlite3
+from datetime import datetime, timezone
+from typing import Any, Optional
+
+logger = logging.getLogger(__name__)
+
+_RETENTION_RULES_TABLE = """
+CREATE TABLE IF NOT EXISTS retention_rules (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    profile_id TEXT NOT NULL,
+    rule_name TEXT NOT NULL,
+    days INTEGER NOT NULL,
+    description TEXT DEFAULT '',
+    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    UNIQUE(profile_id, rule_name)
+)
+"""
+
+_FACTS_TABLE_CHECK = """
+SELECT name FROM sqlite_master
+WHERE type='table' AND name='atomic_facts'
+"""
+
+
+class RetentionEngine:
+    """Named retention rules for compliance (GDPR, HIPAA, custom).
+
+    Rules are bound to profiles. Each rule specifies a retention period.
+    The engine can identify expired facts and enforce deletion.
+    """
+
+    def __init__(self, db: sqlite3.Connection) -> None:
+        self._db = db
+        self._ensure_table()
+
+    # ------------------------------------------------------------------
+    # Internal helpers
+    # ------------------------------------------------------------------
+
+    def _ensure_table(self) -> None:
+        """Create the retention_rules table if it does not exist."""
+        self._db.execute(_RETENTION_RULES_TABLE)
+        self._db.commit()
+
+    def _has_facts_table(self) -> bool:
+        """Check if atomic_facts table exists in the database."""
+        row = self._db.execute(_FACTS_TABLE_CHECK).fetchone()
+        return row is not None
+
+    @staticmethod
+    def _age_in_days(created_at_str: str) -> float:
+        """Calculate age from an ISO timestamp string to now, in days."""
+        try:
+            created = datetime.fromisoformat(created_at_str)
+            if created.tzinfo is None:
+                created = created.replace(tzinfo=timezone.utc)
+            now = datetime.now(timezone.utc)
+            return (now - created).total_seconds() / 86400.0
+        except (ValueError, TypeError):
+            return 0.0
+
+    # ------------------------------------------------------------------
+    # Rule management
+    # ------------------------------------------------------------------
+
+    def add_rule(
+        self,
+        profile_id: str,
+        rule_name: str,
+        days: int,
+        description: str = "",
+    ) -> None:
+        """Add a retention rule to a profile.
+
+        Args:
+            profile_id: Profile this rule applies to.
+            rule_name: Human-readable name (e.g. 'GDPR-30d').
+            days: Retention period in days.
+            description: Optional description of the rule.
+
+        Raises:
+            sqlite3.IntegrityError: If rule_name already exists for profile.
+        """
+        self._db.execute(
+            "INSERT OR REPLACE INTO retention_rules "
+            "(profile_id, rule_name, days, description) "
+            "VALUES (?, ?, ?, ?)",
+            (profile_id, rule_name, days, description),
+        )
+        self._db.commit()
+        logger.info(
+            "Added retention rule '%s' (%d days) to profile '%s'",
+            rule_name, days, profile_id,
+        )
+
+    def remove_rule(self, profile_id: str, rule_name: str) -> None:
+        """Remove a retention rule.
+
+        Args:
+            profile_id: Profile the rule belongs to.
+            rule_name: Name of the rule to remove.
+        """
+        self._db.execute(
+            "DELETE FROM retention_rules "
+            "WHERE profile_id = ? AND rule_name = ?",
+            (profile_id, rule_name),
+        )
+        self._db.commit()
+        logger.info(
+            "Removed retention rule '%s' from profile '%s'",
+            rule_name, profile_id,
+        )
+
+    def get_rules(self, profile_id: str) -> list[dict[str, Any]]:
+        """Get all retention rules for a profile.
+
+        Args:
+            profile_id: Profile to query rules for.
+
+        Returns:
+            List of rule dicts with keys: rule_name, days, description.
+        """
+        rows = self._db.execute(
+            "SELECT rule_name, days, description, created_at "
+            "FROM retention_rules WHERE profile_id = ? "
+            "ORDER BY id",
+            (profile_id,),
+        ).fetchall()
+        return [
+            {
+                "rule_name": r[0],
+                "days": r[1],
+                "description": r[2],
+                "created_at": r[3],
+            }
+            for r in rows
+        ]
+
+    # ------------------------------------------------------------------
+    # Expiration detection
+    # ------------------------------------------------------------------
+
+    def get_expired_facts(self, profile_id: str) -> list[str]:
+        """Get fact IDs that have exceeded their retention period.
+
+        Checks each fact's created_at against the profile's shortest
+        retention rule. A fact is expired if its age exceeds the
+        minimum retention days across all rules for the profile.
+
+        Args:
+            profile_id: Profile to check facts for.
+
+        Returns:
+            List of expired fact IDs (as strings).
+        """
+        rules = self.get_rules(profile_id)
+        if not rules:
+            return []
+
+        # Use the shortest retention period (most restrictive)
+        min_days = min(r["days"] for r in rules)
+
+        if not self._has_facts_table():
+            return []
+
+        rows = self._db.execute(
+            "SELECT id, created_at FROM atomic_facts "
+            "WHERE profile_id = ?",
+            (profile_id,),
+        ).fetchall()
+
+        expired: list[str] = []
+        for row in rows:
+            fact_id = str(row[0])
+            created_at = row[1] if len(row) > 1 else None
+            if created_at and self._age_in_days(created_at) > min_days:
+                expired.append(fact_id)
+
+        return expired
+
+    # ------------------------------------------------------------------
+    # Enforcement
+    # ------------------------------------------------------------------
+
+    def enforce(self, profile_id: str) -> dict[str, Any]:
+        """Enforce retention rules — delete expired facts.
+
+        Finds all expired facts for the profile and deletes them.
+
+        Args:
+            profile_id: Profile to enforce rules on.
+
+        Returns:
+            Dict with keys: deleted_count, expired_ids, profile_id.
+        """
+        expired_ids = self.get_expired_facts(profile_id)
+        deleted_count = 0
+
+        if expired_ids and self._has_facts_table():
+            placeholders = ",".join("?" for _ in expired_ids)
+            self._db.execute(
+                f"DELETE FROM atomic_facts WHERE id IN ({placeholders})",
+                [int(fid) for fid in expired_ids],
+            )
+            self._db.commit()
+            deleted_count = len(expired_ids)
+            logger.info(
+                "Retention enforcement: deleted %d facts from profile '%s'",
+                deleted_count, profile_id,
+            )
+
+        return {
+            "profile_id": profile_id,
+            "deleted_count": deleted_count,
+            "expired_ids": expired_ids,
+        }