superlocalmemory 4.1.13 → 4.1.14

package/src/superlocalmemory/compliance/lifecycle.py

@@ -1,158 +0,0 @@
-# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
-# Licensed under AGPL-3.0-or-later - see LICENSE file
-# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
-
-"""SuperLocalMemory V3 — Memory Lifecycle Management.
-
-Implements Active → Warm → Cold → Archived state machine.
-Coupled with Langevin dynamics: positions naturally create lifecycle states.
-
-Ported from V2.8 with Langevin coupling (Innovation's unique feature).
-
-Part of Qualixar | Author: Varun Pratap Bhardwaj
-"""
-
-from __future__ import annotations
-
-import logging
-from datetime import UTC, datetime, timedelta
-
-from superlocalmemory.storage.models import AtomicFact, MemoryLifecycle
-
-logger = logging.getLogger(__name__)
-
-# Lifecycle transition thresholds (days since last access)
-_ACTIVE_MAX_DAYS = 7        # Active for 7 days after last access
-_WARM_MAX_DAYS = 30         # Warm for 30 days
-_COLD_MAX_DAYS = 90         # Cold for 90 days, then archived
-
-# Langevin weight thresholds (if Langevin is available)
-_ACTIVE_WEIGHT_MIN = 0.7
-_WARM_WEIGHT_MIN = 0.4
-_COLD_WEIGHT_MIN = 0.1
-
-
-class LifecycleManager:
-    """Manage memory lifecycle states.
-
-    Two complementary strategies:
-    1. Time-based: days since last access → state transition
-    2. Langevin-based: position on Poincaré ball → lifecycle weight → state
-
-    When Langevin is available, it takes precedence (more nuanced).
-    Time-based is the fallback for Mode A (no dynamics).
-    """
-
-    def __init__(self, db, langevin=None) -> None:
-        self._db = db
-        self._langevin = langevin
-
-    def get_lifecycle_state(self, fact: AtomicFact) -> MemoryLifecycle:
-        """Determine current lifecycle state for a fact."""
-        # Strategy 1: Langevin-based (if available and position exists)
-        if self._langevin is not None and fact.langevin_position:
-            weight = self._langevin.compute_lifecycle_weight(fact.langevin_position)
-            return self._langevin.get_lifecycle_state(weight)
-
-        # Strategy 2: Time-based fallback
-        return self._time_based_state(fact)
-
-    def update_lifecycle(self, fact_id: str, profile_id: str) -> MemoryLifecycle:
-        """Recompute and persist lifecycle state for a fact."""
-        rows = self._db.execute(
-            "SELECT lifecycle, access_count, created_at FROM atomic_facts "
-            "WHERE fact_id = ? AND profile_id = ?",
-            (fact_id, profile_id),
-        )
-        if not rows:
-            return MemoryLifecycle.ACTIVE
-
-        d = dict(rows[0])
-        days = self._days_since(d.get("created_at", ""))
-        access = d.get("access_count", 0)
-
-        if access > 10 or days < _ACTIVE_MAX_DAYS:
-            state = MemoryLifecycle.ACTIVE
-        elif days < _WARM_MAX_DAYS:
-            state = MemoryLifecycle.WARM
-        elif days < _COLD_MAX_DAYS:
-            state = MemoryLifecycle.COLD
-        else:
-            state = MemoryLifecycle.ARCHIVED
-
-        current = d.get("lifecycle", "active")
-        if current != state.value:
-            self._db.update_fact(fact_id, {"lifecycle": state})
-            logger.debug("Fact %s: %s → %s", fact_id, current, state.value)
-
-        return state
-
-    def run_maintenance(self, profile_id: str) -> dict[str, int]:
-        """Run lifecycle maintenance on all facts in a profile.
-
-        If Langevin is available, evolve positions and update states.
-        Otherwise, use time-based transitions.
-        """
-        facts = self._db.get_all_facts(profile_id)
-        counts: dict[str, int] = {s.value: 0 for s in MemoryLifecycle}
-        transitions = 0
-
-        for fact in facts:
-            old_state = fact.lifecycle
-
-            if self._langevin is not None and fact.langevin_position:
-                # Langevin step
-                age = self._days_since(fact.created_at)
-                new_pos, weight = self._langevin.step(
-                    fact.langevin_position, fact.access_count, age, fact.importance
-                )
-                new_state = self._langevin.get_lifecycle_state(weight)
-                # Persist position + state
-                self._db.update_fact(fact.fact_id, {
-                    "langevin_position": new_pos,
-                    "lifecycle": new_state,
-                })
-            else:
-                new_state = self._time_based_state(fact)
-                if new_state != old_state:
-                    self._db.update_fact(fact.fact_id, {"lifecycle": new_state})
-
-            counts[new_state.value] = counts.get(new_state.value, 0) + 1
-            if new_state != old_state:
-                transitions += 1
-
-        counts["transitions"] = transitions
-        logger.info("Lifecycle maintenance for '%s': %s", profile_id, counts)
-        return counts
-
-    def get_archived_facts(self, profile_id: str) -> list[AtomicFact]:
-        """Get all archived facts (candidates for deletion/export)."""
-        rows = self._db.execute(
-            "SELECT * FROM atomic_facts WHERE profile_id = ? AND lifecycle = 'archived'",
-            (profile_id,),
-        )
-        return [self._db._row_to_fact(r) for r in rows]
-
-    # -- Internal ----------------------------------------------------------
-
-    def _time_based_state(self, fact: AtomicFact) -> MemoryLifecycle:
-        """Determine lifecycle state from time since creation + access count."""
-        days = self._days_since(fact.created_at)
-        if fact.access_count > 10 or days < _ACTIVE_MAX_DAYS:
-            return MemoryLifecycle.ACTIVE
-        if days < _WARM_MAX_DAYS:
-            return MemoryLifecycle.WARM
-        if days < _COLD_MAX_DAYS:
-            return MemoryLifecycle.COLD
-        return MemoryLifecycle.ARCHIVED
-
-    @staticmethod
-    def _days_since(iso_date: str) -> float:
-        """Days since an ISO date string."""
-        if not iso_date:
-            return 0.0
-        try:
-            dt = datetime.fromisoformat(iso_date)
-            return (datetime.now(UTC) - dt).total_seconds() / 86400.0
-        except (ValueError, TypeError):
-            return 0.0

package/src/superlocalmemory/compliance/retention.py

@@ -1,415 +0,0 @@
-# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
-# Licensed under AGPL-3.0-or-later - see LICENSE file
-# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
-
-"""Named retention rules engine for compliance (GDPR, HIPAA, custom).
-
-Rules are bound to profiles. Each rule specifies a retention period
-in days. The engine can identify expired facts and enforce deletion.
-
-Retention rules are stored in a dedicated SQLite table and operate
-independently of the main memory lifecycle system.
-"""
-
-from __future__ import annotations
-
-import json
-import logging
-import sqlite3
-from datetime import datetime, timezone
-from typing import Any
-
-logger = logging.getLogger(__name__)
-
-_RETENTION_RULES_TABLE = """
-CREATE TABLE IF NOT EXISTS retention_rules (
-    id INTEGER PRIMARY KEY AUTOINCREMENT,
-    profile_id TEXT NOT NULL,
-    rule_name TEXT NOT NULL,
-    days INTEGER NOT NULL,
-    description TEXT DEFAULT '',
-    framework TEXT NOT NULL DEFAULT 'custom',
-    action TEXT NOT NULL DEFAULT 'archive',
-    applies_to TEXT NOT NULL DEFAULT '{}',
-    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
-    UNIQUE(profile_id, rule_name)
-)
-"""
-
-# Terminal lifecycle zone each action moves an expired fact into. 'notify'
-# changes nothing — it only surfaces the count so an operator can act.
-# atomic_facts.lifecycle CHECK allows only active/warm/cold/archived, so both
-# retention actions land in the 'archived' lifecycle zone. 'tombstone' is
-# additionally flagged purgeable via archive_status so a purge job can find it —
-# this avoids a risky rebuild of the (large) atomic_facts CHECK constraint.
-_ACTION_LIFECYCLE = {"archive": "archived", "tombstone": "archived"}
-_TOMBSTONE_ACTIONS = frozenset({"tombstone"})
-_VALID_ACTIONS = ("archive", "tombstone", "notify")
-_SUPPORTED_APPLIES_TO = frozenset({"scope", "fact_type", "signal_type", "session_id"})
-
-_FACTS_TABLE_CHECK = """
-SELECT name FROM sqlite_master
-WHERE type='table' AND name='atomic_facts'
-"""
-
-
-class RetentionEngine:
-    """Named retention rules for compliance (GDPR, HIPAA, custom).
-
-    Rules are bound to profiles. Each rule specifies a retention period.
-    The engine can identify expired facts and enforce deletion.
-    """
-
-    def __init__(self, db: sqlite3.Connection, *, autocommit: bool = True) -> None:
-        self._db = db
-        self._autocommit = autocommit
-        self._ensure_table()
-
-    # ------------------------------------------------------------------
-    # Internal helpers
-    # ------------------------------------------------------------------
-
-    def _ensure_table(self) -> None:
-        """Create the retention_rules table, self-migrating older schemas.
-
-        Pre-GDPR-model tables had only (rule_name, days, description). Add the
-        framework/action/applies_to columns in place so existing rules keep
-        working (defaults: framework='custom', action='archive').
-        """
-        self._db.execute(_RETENTION_RULES_TABLE)
-        cols = {r[1] for r in self._db.execute(
-            "PRAGMA table_info(retention_rules)").fetchall()}
-        for col, ddl in (
-            ("framework", "framework TEXT NOT NULL DEFAULT 'custom'"),
-            ("action", "action TEXT NOT NULL DEFAULT 'archive'"),
-            ("applies_to", "applies_to TEXT NOT NULL DEFAULT '{}'"),
-        ):
-            if col not in cols:
-                self._db.execute(f"ALTER TABLE retention_rules ADD COLUMN {ddl}")
-        self._commit_if_owned()
-
-    def _commit_if_owned(self) -> None:
-        """Commit only when this engine owns the transaction boundary."""
-        if self._autocommit:
-            self._db.commit()
-
-    def _has_facts_table(self) -> bool:
-        """Check if atomic_facts table exists in the database."""
-        row = self._db.execute(_FACTS_TABLE_CHECK).fetchone()
-        return row is not None
-
-    @staticmethod
-    def _age_in_days(created_at_str: str) -> float:
-        """Calculate age from an ISO timestamp string to now, in days."""
-        try:
-            created = datetime.fromisoformat(created_at_str)
-            if created.tzinfo is None:
-                created = created.replace(tzinfo=timezone.utc)
-            now = datetime.now(timezone.utc)
-            return (now - created).total_seconds() / 86400.0
-        except (ValueError, TypeError):
-            return 0.0
-
-    # ------------------------------------------------------------------
-    # Rule management
-    # ------------------------------------------------------------------
-
-    def add_rule(
-        self,
-        profile_id: str,
-        rule_name: str,
-        days: int,
-        description: str = "",
-    ) -> None:
-        """Add a retention rule to a profile.
-
-        Args:
-            profile_id: Profile this rule applies to.
-            rule_name: Human-readable name (e.g. 'GDPR-30d').
-            days: Retention period in days.
-            description: Optional description of the rule.
-
-        Raises:
-            sqlite3.IntegrityError: If rule_name already exists for profile.
-        """
-        self._db.execute(
-            "INSERT OR REPLACE INTO retention_rules "
-            "(profile_id, rule_name, days, description) "
-            "VALUES (?, ?, ?, ?)",
-            (profile_id, rule_name, days, description),
-        )
-        self._commit_if_owned()
-        logger.info(
-            "Added retention rule '%s' (%d days) to profile '%s'",
-            rule_name, days, profile_id,
-        )
-
-    def remove_rule(self, profile_id: str, rule_name: str) -> None:
-        """Remove a retention rule.
-
-        Args:
-            profile_id: Profile the rule belongs to.
-            rule_name: Name of the rule to remove.
-        """
-        self._db.execute(
-            "DELETE FROM retention_rules "
-            "WHERE profile_id = ? AND rule_name = ?",
-            (profile_id, rule_name),
-        )
-        self._commit_if_owned()
-        logger.info(
-            "Removed retention rule '%s' from profile '%s'",
-            rule_name, profile_id,
-        )
-
-    def get_rules(self, profile_id: str) -> list[dict[str, Any]]:
-        """Get all retention rules for a profile (full GDPR shape)."""
-        rows = self._db.execute(
-            "SELECT id, rule_name, days, description, framework, action, "
-            "applies_to, created_at FROM retention_rules "
-            "WHERE profile_id = ? ORDER BY id",
-            (profile_id,),
-        ).fetchall()
-        out: list[dict[str, Any]] = []
-        for r in rows:
-            try:
-                applies = json.loads(r[6]) if r[6] else {}
-            except (ValueError, TypeError):
-                applies = {}
-            out.append({
-                "id": r[0],
-                "name": r[1],
-                "rule_name": r[1],          # legacy alias
-                "retention_days": r[2],
-                "days": r[2],               # legacy alias
-                "description": r[3],
-                "framework": r[4],
-                "action": r[5],
-                "applies_to": applies,
-                "created_at": r[7],
-            })
-        return out
-
-    # ------------------------------------------------------------------
-    # GDPR-model API (used by the compliance route + dashboard)
-    # ------------------------------------------------------------------
-
-    def create_rule(
-        self,
-        *,
-        name: str,
-        framework: str,
-        retention_days: int,
-        action: str,
-        applies_to: dict | None,
-        profile_id: str,
-    ) -> int:
-        """Create a retention rule and return its row id.
-
-        ``action`` is one of archive | tombstone | notify. Raises ValueError on
-        an invalid action so the route returns a clear 4xx rather than storing
-        an unenforceable rule.
-        """
-        if action not in _VALID_ACTIONS:
-            raise ValueError(f"action must be one of {_VALID_ACTIONS}")
-        selectors = applies_to or {}
-        if not isinstance(selectors, dict):
-            raise ValueError("applies_to must be an object")
-        unsupported = set(selectors) - _SUPPORTED_APPLIES_TO
-        if unsupported:
-            raise ValueError(
-                "unsupported applies_to selectors: " + ", ".join(sorted(unsupported))
-            )
-        cur = self._db.execute(
-            "INSERT OR REPLACE INTO retention_rules "
-            "(profile_id, rule_name, days, description, framework, action, applies_to) "
-            "VALUES (?, ?, ?, ?, ?, ?, ?)",
-            (profile_id, name, int(retention_days), "", framework, action,
-             json.dumps(selectors, sort_keys=True)),
-        )
-        self._commit_if_owned()
-        logger.info(
-            "Created retention rule '%s' (%dd, %s/%s) for profile '%s'",
-            name, retention_days, framework, action, profile_id,
-        )
-        return int(cur.lastrowid or 0)
-
-    def list_rules(self, profile_id: str | None = None) -> list[dict[str, Any]]:
-        """List rules for one profile, or all rules when profile_id is None."""
-        if profile_id is not None:
-            return self.get_rules(profile_id)
-        rows = self._db.execute(
-            "SELECT DISTINCT profile_id FROM retention_rules"
-        ).fetchall()
-        out: list[dict[str, Any]] = []
-        for r in rows:
-            out.extend(self.get_rules(r[0]))
-        return out
-
-    def delete_rule(self, profile_id: str, name: str) -> bool:
-        """Delete a rule by (profile_id, name). Returns True if a row was removed."""
-        cur = self._db.execute(
-            "DELETE FROM retention_rules WHERE profile_id = ? AND rule_name = ?",
-            (profile_id, name),
-        )
-        self._commit_if_owned()
-        return cur.rowcount > 0
-
-    # ------------------------------------------------------------------
-    # Expiration detection
-    # ------------------------------------------------------------------
-
-    def get_expired_facts(self, profile_id: str) -> list[str]:
-        """Get fact IDs that have exceeded their retention period.
-
-        Checks each fact's created_at against the profile's shortest
-        retention rule. A fact is expired if its age exceeds the
-        minimum retention days across all rules for the profile.
-
-        Args:
-            profile_id: Profile to check facts for.
-
-        Returns:
-            List of expired fact IDs (as strings).
-        """
-        rules = self.get_rules(profile_id)
-        if not rules:
-            return []
-
-        # Use the shortest retention period (most restrictive)
-        min_days = min(r["retention_days"] for r in rules)
-
-        if not self._has_facts_table():
-            return []
-
-        # atomic_facts is keyed by fact_id (TEXT hex), NOT an integer id.
-        rows = self._db.execute(
-            "SELECT fact_id, created_at FROM atomic_facts WHERE profile_id = ?",
-            (profile_id,),
-        ).fetchall()
-
-        expired: list[str] = []
-        for row in rows:
-            fact_id = str(row[0])
-            created_at = row[1] if len(row) > 1 else None
-            if created_at and self._age_in_days(created_at) > min_days:
-                expired.append(fact_id)
-
-        return expired
-
-    # ------------------------------------------------------------------
-    # Enforcement
-    # ------------------------------------------------------------------
-
-    def enforce(self, profile_id: str) -> dict[str, Any]:
-        """Atomically enforce all rules for one profile."""
-        if not self._autocommit and not self._db.in_transaction:
-            self._db.execute("BEGIN")
-        savepoint = "slm_retention_profile"
-        self._db.execute(f"SAVEPOINT {savepoint}")
-        try:
-            result = self._enforce_uncommitted(profile_id)
-            self._db.execute(f"RELEASE SAVEPOINT {savepoint}")
-            if self._autocommit:
-                self._db.commit()
-            return result
-        except Exception:
-            self._db.execute(f"ROLLBACK TO SAVEPOINT {savepoint}")
-            self._db.execute(f"RELEASE SAVEPOINT {savepoint}")
-            raise
-
-    def _enforce_uncommitted(self, profile_id: str) -> dict[str, Any]:
-        """Enforce every retention rule for a profile, honoring each action.
-
-        For each rule, facts in the profile older than its retention_days are
-        moved to the rule's terminal lifecycle zone:
-          * archive   → lifecycle 'archived'  (retained, hidden from recall)
-          * tombstone → lifecycle 'tombstoned' (soft-deleted, purgeable)
-          * notify    → unchanged (only counted, so an operator can act)
-
-        GDPR-safe: soft-state transitions, never a raw DELETE — erasure is an
-        explicit separate operation. All updates are scoped to ``profile_id``
-        and keyed by ``fact_id`` (TEXT), fixing the prior ``id``/int() bug that
-        made enforcement crash outright.
-
-        Returns per-action counts + the affected fact ids.
-        """
-        rules = self.get_rules(profile_id)
-        result: dict[str, Any] = {
-            "profile_id": profile_id,
-            "archived": 0, "tombstoned": 0, "notified": 0,
-            "deleted_count": 0,  # legacy alias (== tombstoned)
-            "affected_ids": [],
-        }
-        if not rules or not self._has_facts_table():
-            return result
-
-        fact_columns = {
-            row[1] for row in self._db.execute("PRAGMA table_info(atomic_facts)").fetchall()
-        }
-        selector_columns = sorted(_SUPPORTED_APPLIES_TO & fact_columns)
-        selected_columns = ["fact_id", "created_at", *selector_columns]
-        rows = self._db.execute(
-            f"SELECT {', '.join(selected_columns)} FROM atomic_facts "
-            "WHERE profile_id = ?",
-            (profile_id,),
-        ).fetchall()
-        facts = [dict(zip(selected_columns, row, strict=True)) for row in rows]
-
-        affected: set[str] = set()
-        for rule in rules:
-            action = rule.get("action", "archive")
-            days = rule.get("retention_days", rule.get("days", 0))
-            selectors = rule.get("applies_to") or {}
-            if set(selectors) - set(selector_columns):
-                logger.warning(
-                    "Retention rule %r skipped unsupported/unavailable selectors: %s",
-                    rule.get("name"),
-                    sorted(set(selectors) - set(selector_columns)),
-                )
-                continue
-            expired = []
-            for fact in facts:
-                created_at = fact.get("created_at")
-                if not created_at or self._age_in_days(created_at) <= days:
-                    continue
-                matches = True
-                for field, expected in selectors.items():
-                    accepted = expected if isinstance(expected, list) else [expected]
-                    if fact.get(field) not in accepted:
-                        matches = False
-                        break
-                if matches:
-                    expired.append(str(fact["fact_id"]))
-            if not expired:
-                continue
-            if action == "notify":
-                result["notified"] += len(expired)
-                affected.update(expired)
-                continue
-            zone = _ACTION_LIFECYCLE.get(action)
-            if zone is None:
-                continue
-            placeholders = ",".join("?" for _ in expired)
-            self._db.execute(
-                f"UPDATE atomic_facts SET lifecycle = ? "
-                f"WHERE profile_id = ? AND fact_id IN ({placeholders})",
-                [zone, profile_id, *expired],
-            )
-            if action in _TOMBSTONE_ACTIONS:
-                # Flag purgeable without violating the lifecycle CHECK.
-                self._db.execute(
-                    f"UPDATE atomic_facts SET archive_status = 'tombstoned' "
-                    f"WHERE profile_id = ? AND fact_id IN ({placeholders})",
-                    [profile_id, *expired],
-                )
-            result["archived" if action == "archive" else "tombstoned"] += len(expired)
-            affected.update(expired)
-
-        result["affected_ids"] = sorted(affected)
-        result["deleted_count"] = result["tombstoned"]  # legacy alias
-        logger.info(
-            "Retention enforcement for '%s': archived=%d tombstoned=%d notified=%d",
-            profile_id, result["archived"], result["tombstoned"], result["notified"],
-        )
-        return result