superlocalmemory 3.7.7 → 3.8.0

package/src/superlocalmemory/compliance/retention.py

@@ -13,6 +13,7 @@ independently of the main memory lifecycle system.
 
 from __future__ import annotations
 
+import json
 import logging
 import sqlite3
 from datetime import datetime, timezone
@@ -27,11 +28,24 @@ CREATE TABLE IF NOT EXISTS retention_rules (
     rule_name TEXT NOT NULL,
     days INTEGER NOT NULL,
     description TEXT DEFAULT '',
+    framework TEXT NOT NULL DEFAULT 'custom',
+    action TEXT NOT NULL DEFAULT 'archive',
+    applies_to TEXT NOT NULL DEFAULT '{}',
     created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
     UNIQUE(profile_id, rule_name)
 )
 """
 
+# Terminal lifecycle zone each action moves an expired fact into. 'notify'
+# changes nothing — it only surfaces the count so an operator can act.
+# atomic_facts.lifecycle CHECK allows only active/warm/cold/archived, so both
+# retention actions land in the 'archived' lifecycle zone. 'tombstone' is
+# additionally flagged purgeable via archive_status so a purge job can find it —
+# this avoids a risky rebuild of the (large) atomic_facts CHECK constraint.
+_ACTION_LIFECYCLE = {"archive": "archived", "tombstone": "archived"}
+_TOMBSTONE_ACTIONS = frozenset({"tombstone"})
+_VALID_ACTIONS = ("archive", "tombstone", "notify")
+
 _FACTS_TABLE_CHECK = """
 SELECT name FROM sqlite_master
 WHERE type='table' AND name='atomic_facts'
@@ -54,8 +68,22 @@ class RetentionEngine:
     # ------------------------------------------------------------------
 
     def _ensure_table(self) -> None:
-        """Create the retention_rules table if it does not exist."""
+        """Create the retention_rules table, self-migrating older schemas.
+
+        Pre-GDPR-model tables had only (rule_name, days, description). Add the
+        framework/action/applies_to columns in place so existing rules keep
+        working (defaults: framework='custom', action='archive').
+        """
         self._db.execute(_RETENTION_RULES_TABLE)
+        cols = {r[1] for r in self._db.execute(
+            "PRAGMA table_info(retention_rules)").fetchall()}
+        for col, ddl in (
+            ("framework", "framework TEXT NOT NULL DEFAULT 'custom'"),
+            ("action", "action TEXT NOT NULL DEFAULT 'archive'"),
+            ("applies_to", "applies_to TEXT NOT NULL DEFAULT '{}'"),
+        ):
+            if col not in cols:
+                self._db.execute(f"ALTER TABLE retention_rules ADD COLUMN {ddl}")
         self._db.commit()
 
     def _has_facts_table(self) -> bool:
@@ -128,29 +156,89 @@ class RetentionEngine:
         )
 
     def get_rules(self, profile_id: str) -> list[dict[str, Any]]:
-        """Get all retention rules for a profile.
+        """Get all retention rules for a profile (full GDPR shape)."""
+        rows = self._db.execute(
+            "SELECT id, rule_name, days, description, framework, action, "
+            "applies_to, created_at FROM retention_rules "
+            "WHERE profile_id = ? ORDER BY id",
+            (profile_id,),
+        ).fetchall()
+        out: list[dict[str, Any]] = []
+        for r in rows:
+            try:
+                applies = json.loads(r[6]) if r[6] else {}
+            except (ValueError, TypeError):
+                applies = {}
+            out.append({
+                "id": r[0],
+                "name": r[1],
+                "rule_name": r[1],          # legacy alias
+                "retention_days": r[2],
+                "days": r[2],               # legacy alias
+                "description": r[3],
+                "framework": r[4],
+                "action": r[5],
+                "applies_to": applies,
+                "created_at": r[7],
+            })
+        return out
 
-        Args:
-            profile_id: Profile to query rules for.
+    # ------------------------------------------------------------------
+    # GDPR-model API (used by the compliance route + dashboard)
+    # ------------------------------------------------------------------
 
-        Returns:
-            List of rule dicts with keys: rule_name, days, description.
+    def create_rule(
+        self,
+        *,
+        name: str,
+        framework: str,
+        retention_days: int,
+        action: str,
+        applies_to: dict | None,
+        profile_id: str,
+    ) -> int:
+        """Create a retention rule and return its row id.
+
+        ``action`` is one of archive | tombstone | notify. Raises ValueError on
+        an invalid action so the route returns a clear 4xx rather than storing
+        an unenforceable rule.
         """
+        if action not in _VALID_ACTIONS:
+            raise ValueError(f"action must be one of {_VALID_ACTIONS}")
+        cur = self._db.execute(
+            "INSERT OR REPLACE INTO retention_rules "
+            "(profile_id, rule_name, days, description, framework, action, applies_to) "
+            "VALUES (?, ?, ?, ?, ?, ?, ?)",
+            (profile_id, name, int(retention_days), "", framework, action,
+             json.dumps(applies_to or {})),
+        )
+        self._db.commit()
+        logger.info(
+            "Created retention rule '%s' (%dd, %s/%s) for profile '%s'",
+            name, retention_days, framework, action, profile_id,
+        )
+        return int(cur.lastrowid or 0)
+
+    def list_rules(self, profile_id: str | None = None) -> list[dict[str, Any]]:
+        """List rules for one profile, or all rules when profile_id is None."""
+        if profile_id is not None:
+            return self.get_rules(profile_id)
         rows = self._db.execute(
-            "SELECT rule_name, days, description, created_at "
-            "FROM retention_rules WHERE profile_id = ? "
-            "ORDER BY id",
-            (profile_id,),
+            "SELECT DISTINCT profile_id FROM retention_rules"
         ).fetchall()
-        return [
-            {
-                "rule_name": r[0],
-                "days": r[1],
-                "description": r[2],
-                "created_at": r[3],
-            }
-            for r in rows
-        ]
+        out: list[dict[str, Any]] = []
+        for r in rows:
+            out.extend(self.get_rules(r[0]))
+        return out
+
+    def delete_rule(self, profile_id: str, name: str) -> bool:
+        """Delete a rule by (profile_id, name). Returns True if a row was removed."""
+        cur = self._db.execute(
+            "DELETE FROM retention_rules WHERE profile_id = ? AND rule_name = ?",
+            (profile_id, name),
+        )
+        self._db.commit()
+        return cur.rowcount > 0
 
     # ------------------------------------------------------------------
     # Expiration detection
@@ -174,14 +262,14 @@ class RetentionEngine:
             return []
 
         # Use the shortest retention period (most restrictive)
-        min_days = min(r["days"] for r in rules)
+        min_days = min(r["retention_days"] for r in rules)
 
         if not self._has_facts_table():
             return []
 
+        # atomic_facts is keyed by fact_id (TEXT hex), NOT an integer id.
         rows = self._db.execute(
-            "SELECT id, created_at FROM atomic_facts "
-            "WHERE profile_id = ?",
+            "SELECT fact_id, created_at FROM atomic_facts WHERE profile_id = ?",
             (profile_id,),
         ).fetchall()
 
@@ -199,34 +287,77 @@ class RetentionEngine:
     # ------------------------------------------------------------------
 
     def enforce(self, profile_id: str) -> dict[str, Any]:
-        """Enforce retention rules — delete expired facts.
+        """Enforce every retention rule for a profile, honoring each action.
 
-        Finds all expired facts for the profile and deletes them.
+        For each rule, facts in the profile older than its retention_days are
+        moved to the rule's terminal lifecycle zone:
+          * archive   → lifecycle 'archived'  (retained, hidden from recall)
+          * tombstone → lifecycle 'tombstoned' (soft-deleted, purgeable)
+          * notify    → unchanged (only counted, so an operator can act)
 
-        Args:
-            profile_id: Profile to enforce rules on.
+        GDPR-safe: soft-state transitions, never a raw DELETE — erasure is an
+        explicit separate operation. All updates are scoped to ``profile_id``
+        and keyed by ``fact_id`` (TEXT), fixing the prior ``id``/int() bug that
+        made enforcement crash outright.
 
-        Returns:
-            Dict with keys: deleted_count, expired_ids, profile_id.
+        Returns per-action counts + the affected fact ids.
         """
-        expired_ids = self.get_expired_facts(profile_id)
-        deleted_count = 0
+        rules = self.get_rules(profile_id)
+        result: dict[str, Any] = {
+            "profile_id": profile_id,
+            "archived": 0, "tombstoned": 0, "notified": 0,
+            "deleted_count": 0,  # legacy alias (== tombstoned)
+            "affected_ids": [],
+        }
+        if not rules or not self._has_facts_table():
+            return result
+
+        rows = self._db.execute(
+            "SELECT fact_id, created_at FROM atomic_facts WHERE profile_id = ?",
+            (profile_id,),
+        ).fetchall()
 
-        if expired_ids and self._has_facts_table():
-            placeholders = ",".join("?" for _ in expired_ids)
+        affected: set[str] = set()
+        for rule in rules:
+            action = rule.get("action", "archive")
+            days = rule.get("retention_days", rule.get("days", 0))
+            expired = [
+                str(r[0]) for r in rows
+                if r[1] and self._age_in_days(r[1]) > days
+            ]
+            if not expired:
+                continue
+            if action == "notify":
+                result["notified"] += len(expired)
+                affected.update(expired)
+                continue
+            zone = _ACTION_LIFECYCLE.get(action)
+            if zone is None:
+                continue
+            placeholders = ",".join("?" for _ in expired)
             self._db.execute(
-                f"DELETE FROM atomic_facts WHERE id IN ({placeholders})",
-                [int(fid) for fid in expired_ids],
-            )
-            self._db.commit()
-            deleted_count = len(expired_ids)
-            logger.info(
-                "Retention enforcement: deleted %d facts from profile '%s'",
-                deleted_count, profile_id,
+                f"UPDATE atomic_facts SET lifecycle = ? "
+                f"WHERE profile_id = ? AND fact_id IN ({placeholders})",
+                [zone, profile_id, *expired],
             )
+            if action in _TOMBSTONE_ACTIONS:
+                # Flag purgeable without violating the lifecycle CHECK.
+                try:
+                    self._db.execute(
+                        f"UPDATE atomic_facts SET archive_status = 'tombstoned' "
+                        f"WHERE profile_id = ? AND fact_id IN ({placeholders})",
+                        [profile_id, *expired],
+                    )
+                except Exception:
+                    pass
+            result["archived" if action == "archive" else "tombstoned"] += len(expired)
+            affected.update(expired)
 
-        return {
-            "profile_id": profile_id,
-            "deleted_count": deleted_count,
-            "expired_ids": expired_ids,
-        }
+        self._db.commit()
+        result["affected_ids"] = sorted(affected)
+        result["deleted_count"] = result["tombstoned"]  # legacy alias
+        logger.info(
+            "Retention enforcement for '%s': archived=%d tombstoned=%d notified=%d",
+            profile_id, result["archived"], result["tombstoned"], result["notified"],
+        )
+        return result

package/src/superlocalmemory/core/backend_orchestrator.py

@@ -17,8 +17,6 @@ Part of Qualixar | Author: Varun Pratap Bhardwaj
 from __future__ import annotations
 
 import logging
-import sqlite3
-import threading
 from pathlib import Path
 from typing import TYPE_CHECKING, Any
 
@@ -366,47 +364,11 @@ class BackendOrchestrator:
             logger.warning("LanceDB init failed: %s", exc)
             self._lancedb = None
 
-    # ------------------------------------------------------------------
-    # Internal: Migration
-    # ------------------------------------------------------------------
-
-    def _migrate_cozo(self) -> None:
-        self._update_status("cozo", "migrating")
-
-        def _run():
-            conn = sqlite3.connect(str(self._data_dir / "memory.db"))
-            conn.execute("PRAGMA journal_mode=WAL")
-            conn.execute("PRAGMA query_only=ON")  # F-07: read-only in migration thread
-            try:
-                count = self._cozo.bulk_import_from_sqlite(conn)
-                self._update_status("cozo", "active", count)
-                logger.info("CozoDB migration complete: %d edges", count)
-            except Exception as exc:
-                logger.error("CozoDB migration failed: %s", exc)
-                self._update_status("cozo", "failed", error=str(exc))
-            finally:
-                conn.close()
-
-        threading.Thread(target=_run, daemon=True).start()
-
-    def _migrate_lancedb(self) -> None:
-        self._update_status("lancedb", "migrating")
-
-        def _run():
-            conn = sqlite3.connect(str(self._data_dir / "memory.db"))
-            conn.execute("PRAGMA journal_mode=WAL")
-            conn.execute("PRAGMA query_only=ON")
-            try:
-                count = self._lancedb.bulk_import_from_sqlite(conn)
-                self._update_status("lancedb", "active", count)
-                logger.info("LanceDB migration complete: %d vectors", count)
-            except Exception as exc:
-                logger.error("LanceDB migration failed: %s", exc)
-                self._update_status("lancedb", "failed", error=str(exc))
-            finally:
-                conn.close()
-
-        threading.Thread(target=_run, daemon=True).start()
+    # v3.7.9 (scale MEDIUM-2): _migrate_cozo/_migrate_lancedb were dead code —
+    # never called anywhere — and bypassed the staged prepare→verify→promote
+    # safety envelope (no fingerprint, no parity, no backup). Removed so a future
+    # caller cannot re-import outside the lifecycle. Emergency re-imports must go
+    # through the Scale Engine lifecycle.
 
     # ------------------------------------------------------------------
     # Internal: Status

package/src/superlocalmemory/core/community_summary.py

@@ -0,0 +1,267 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under AGPL-3.0-or-later - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""Community summaries (Wave Q2) — one synthesized report per entity community.
+
+Rides on the entity-community backbone (core.entity_community). For each
+community it gathers the member entities' facts, EXCLUDES superseded facts
+(bi-temporal — market CRIT-3), then produces:
+
+  - a keyword-dense signal (always, Mode A, zero-LLM), and
+  - a summary: Mode B/C LLM synthesis via the shared core.Summarizer
+    (which itself falls back to a heuristic), else the Mode A keyword-dense
+    line. Fail-open — a summarizer error never breaks generation.
+
+Surfacing is on-device-safe: summaries are PRECOMPUTED here in the background
+and later matched to a query as a single thematic-context block (Q2b) — never
+a GraphRAG-style per-query LLM fan-out (market CRIT-1). member_fact_ids gives
+drill-down back to the source atoms (Q3).
+
+Part of Qualixar | Author: Varun Pratap Bhardwaj
+License: AGPL-3.0-or-later
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from collections import Counter, defaultdict
+from typing import Any
+
+from superlocalmemory.core.entity_community import EntityCommunityBuilder
+
+logger = logging.getLogger(__name__)
+
+_STOPWORDS = frozenset({
+    "the", "a", "an", "is", "was", "were", "are", "be", "been", "being",
+    "have", "has", "had", "do", "does", "did", "will", "would", "could",
+    "should", "may", "might", "shall", "can", "to", "of", "in", "for", "on",
+    "with", "at", "by", "from", "as", "into", "through", "and", "but", "or",
+    "not", "no", "this", "that", "these", "those", "it", "its", "they",
+    "them", "their", "he", "she", "his", "her", "we", "our", "you", "your",
+    "i", "my", "me", "his", "was", "who", "what", "when", "where", "how",
+})
+
+
+class CommunitySummaryBuilder:
+    """Generate + persist one summary per entity community (background)."""
+
+    def __init__(
+        self,
+        db: Any,
+        summarizer: Any = None,
+        max_communities: int = 50,
+        min_facts: int = 2,
+        max_facts_per_community: int = 30,
+        max_keywords: int = 8,
+        summary_max_chars: int = 512,
+    ) -> None:
+        self._db = db
+        self._summarizer = summarizer
+        self._max_communities = max(1, int(max_communities))
+        self._min_facts = max(1, int(min_facts))
+        self._max_facts = max(1, int(max_facts_per_community))
+        self._max_keywords = max(1, int(max_keywords))
+        self._summary_max_chars = max(64, int(summary_max_chars))
+
+    # ------------------------------------------------------------------
+    # Generation
+    # ------------------------------------------------------------------
+
+    def compute_and_store(self, profile_id: str) -> dict[str, int]:
+        communities = EntityCommunityBuilder(self._db).get_communities(profile_id)
+        try:
+            self._db.execute(
+                "DELETE FROM community_summaries WHERE profile_id = ?",
+                (profile_id,),
+            )
+        except Exception as exc:
+            logger.debug("community_summaries clear failed: %s", exc)
+        if not communities:
+            return {"summaries_written": 0, "communities": 0}
+
+        entity_to_cid: dict[str, int] = {
+            e: cid for cid, ents in communities.items() for e in ents
+        }
+        cid_facts = self._gather_facts(profile_id, entity_to_cid)
+        name_map = self._entity_names(profile_id, communities)
+
+        written = 0
+        ordered = sorted(
+            cid_facts.items(), key=lambda kv: len(kv[1]), reverse=True,
+        )
+        for cid, facts in ordered:
+            if written >= self._max_communities:
+                break
+            seen: set[str] = set()
+            vf: list[tuple[str, str]] = []
+            for fid, content in facts:
+                if fid in seen:
+                    continue
+                seen.add(fid)
+                vf.append((fid, content))
+            if len(vf) < self._min_facts:
+                continue
+
+            fact_ids = [fid for fid, _ in vf]
+            contents = [c for _, c in vf][: self._max_facts]
+            entity_ids = list(communities.get(cid, []))
+            entity_names = [name_map.get(e, e) for e in entity_ids]
+            keywords = self._keywords(contents)
+            summary = self._summary(contents, entity_names, keywords)
+
+            try:
+                self._db.execute(
+                    "INSERT OR REPLACE INTO community_summaries "
+                    "(profile_id, community_id, summary, keywords, "
+                    " entity_ids_json, fact_ids_json, fact_count, computed_at) "
+                    "VALUES (?, ?, ?, ?, ?, ?, ?, datetime('now'))",
+                    (
+                        profile_id, cid, summary, keywords,
+                        json.dumps(entity_ids), json.dumps(fact_ids),
+                        len(fact_ids),
+                    ),
+                )
+                written += 1
+            except Exception as exc:
+                logger.debug("community_summaries write failed (%s): %s", cid, exc)
+
+        return {"summaries_written": written, "communities": len(communities)}
+
+    # ------------------------------------------------------------------
+    # Read API
+    # ------------------------------------------------------------------
+
+    def get_summaries(self, profile_id: str) -> list[dict]:
+        try:
+            rows = self._db.execute(
+                "SELECT * FROM community_summaries WHERE profile_id = ? "
+                "ORDER BY fact_count DESC",
+                (profile_id,),
+            )
+        except Exception as exc:
+            logger.debug("get_summaries failed: %s", exc)
+            return []
+        return [dict(r) for r in rows]
+
+    def get_summary(self, profile_id: str, community_id: int) -> dict | None:
+        try:
+            rows = self._db.execute(
+                "SELECT * FROM community_summaries "
+                "WHERE profile_id = ? AND community_id = ?",
+                (profile_id, int(community_id)),
+            )
+        except Exception as exc:
+            logger.debug("get_summary failed: %s", exc)
+            return None
+        return dict(rows[0]) if rows else None
+
+    # ------------------------------------------------------------------
+    # Internal
+    # ------------------------------------------------------------------
+
+    def _gather_facts(
+        self, profile_id: str, entity_to_cid: dict[str, int],
+    ) -> dict[int, list[tuple[str, str]]]:
+        """One scan → {community_id -> [(fact_id, content)]}, superseded dropped."""
+        try:
+            rows = self._db.execute(
+                "SELECT fact_id, canonical_entities_json, content "
+                "FROM atomic_facts WHERE profile_id = ?",
+                (profile_id,),
+            )
+        except Exception as exc:
+            logger.debug("community fact scan failed: %s", exc)
+            return {}
+
+        cid_facts: dict[int, list[tuple[str, str]]] = defaultdict(list)
+        for row in rows:
+            d = dict(row)
+            raw = d.get("canonical_entities_json")
+            if not raw:
+                continue
+            try:
+                ents = json.loads(raw)
+            except (ValueError, TypeError):
+                continue
+            if not isinstance(ents, list):
+                continue
+            cids = {
+                entity_to_cid[str(e).strip()]
+                for e in ents
+                if str(e).strip() in entity_to_cid
+            }
+            if not cids:
+                continue
+            fid = str(d["fact_id"])
+            content = d.get("content") or ""
+            for cid in cids:
+                cid_facts[cid].append((fid, content))
+
+        all_fids = list({fid for lst in cid_facts.values() for fid, _ in lst})
+        invalid: set[str] = set()
+        if all_fids:
+            try:
+                invalid = self._db.get_invalidated_fact_ids(all_fids, profile_id)
+            except Exception as exc:
+                logger.debug("invalidated-fact lookup failed: %s", exc)
+        if invalid:
+            cid_facts = {
+                cid: [(fid, c) for fid, c in lst if fid not in invalid]
+                for cid, lst in cid_facts.items()
+            }
+        return cid_facts
+
+    def _entity_names(
+        self, profile_id: str, communities: dict[int, list[str]],
+    ) -> dict[str, str]:
+        all_eids = list({e for ents in communities.values() for e in ents})
+        name_map: dict[str, str] = {}
+        chunk = 900
+        for start in range(0, len(all_eids), chunk):
+            batch = all_eids[start:start + chunk]
+            ph = ",".join("?" for _ in batch)
+            try:
+                rows = self._db.execute(
+                    "SELECT entity_id, canonical_name FROM canonical_entities "
+                    f"WHERE profile_id = ? AND entity_id IN ({ph})",
+                    (profile_id, *batch),
+                )
+            except Exception as exc:
+                logger.debug("entity-name lookup failed: %s", exc)
+                continue
+            for r in rows:
+                d = dict(r)
+                name_map[str(d["entity_id"])] = str(d.get("canonical_name") or "")
+        return name_map
+
+    def _keywords(self, contents: list[str]) -> str:
+        tokens: list[str] = []
+        for text in contents:
+            for word in text.lower().split():
+                w = word.strip(".,;:!?\"'()[]{}")
+                if len(w) > 2 and w not in _STOPWORDS:
+                    tokens.append(w)
+        top = [w for w, _ in Counter(tokens).most_common(self._max_keywords)]
+        return ", ".join(top)
+
+    def _summary(
+        self, contents: list[str], entity_names: list[str], keywords: str,
+    ) -> str:
+        if self._summarizer is not None:
+            try:
+                text = self._summarizer.summarize_cluster(
+                    [{"content": c} for c in contents],
+                )
+                if text and text.strip():
+                    return text.strip()[: self._summary_max_chars]
+            except Exception as exc:
+                logger.debug("community summarizer failed (fail-open): %s", exc)
+        # Mode A keyword-dense fallback.
+        names = [n for n in entity_names if n]
+        topic = ", ".join(names[:6]) if names else "related memories"
+        base = f"Topics: {topic}."
+        if keywords:
+            base += f" Key terms: {keywords}."
+        return base[: self._summary_max_chars]