superlocalmemory 3.7.7 → 3.8.0

package/src/superlocalmemory/cli/pending_store.py

@@ -43,6 +43,7 @@ _MAX_RETRY_DELAY_SECONDS = 3600
 _SCHEMA = """
 CREATE TABLE IF NOT EXISTS pending_memories (
     id INTEGER PRIMARY KEY AUTOINCREMENT,
+    profile_id TEXT NOT NULL DEFAULT 'default',
     content TEXT NOT NULL,
     tags TEXT DEFAULT '',
     metadata TEXT DEFAULT '{}',
@@ -62,6 +63,12 @@ def _get_db(base_dir: Path | None = None) -> sqlite3.Connection:
     db_path = d / _PENDING_DB
     conn = sqlite3.connect(str(db_path), timeout=5)
     conn.execute("PRAGMA journal_mode=WAL")
+    # C4: pending queue can hold not-yet-materialized memory content owner-only.
+    try:
+        from superlocalmemory.core.security_primitives import harden_db_perms
+        harden_db_perms(db_path)
+    except Exception:
+        pass
     conn.execute(_SCHEMA)
     columns = {
         row[1]
@@ -73,6 +80,15 @@ def _get_db(base_dir: Path | None = None) -> sqlite3.Connection:
             "next_retry_at REAL DEFAULT 0"
         )
         conn.commit()
+    # Per-profile isolation: a queued item must materialize under the profile
+    # that was active when it was enqueued — never under whatever profile is
+    # active at drain time. Existing rows backfill to 'default'.
+    if "profile_id" not in columns:
+        conn.execute(
+            "ALTER TABLE pending_memories ADD COLUMN "
+            "profile_id TEXT NOT NULL DEFAULT 'default'"
+        )
+        conn.commit()
     # Pre-V3.7 rows were terminally hidden after three failures. Restore them
     # to the retry queue; M018 makes replay idempotent and no raw evidence may
     # remain stranded solely because an older version exhausted its counter.
@@ -89,8 +105,12 @@ def store_pending(
     tags: str = "",
     metadata: dict | None = None,
     base_dir: Path | None = None,
+    profile_id: str = "default",
 ) -> int:
-    """Store content in pending table. Returns the row ID.
+    """Store content in pending table under a profile. Returns the row ID.
+
+    ``profile_id`` captures the profile active at ENQUEUE time so a later
+    profile switch can never redirect this memory to a different profile.
 
     This is intentionally FAST — no engine init, no embedding, no model loading.
     Just a raw SQLite INSERT (~0.1s).
@@ -98,9 +118,11 @@ def store_pending(
     conn = _get_db(base_dir)
     try:
         cur = conn.execute(
-            "INSERT INTO pending_memories (content, tags, metadata, created_at, status) "
-            "VALUES (?, ?, ?, ?, 'pending')",
-            (content, tags, json.dumps(metadata or {}), time.strftime("%Y-%m-%dT%H:%M:%S")),
+            "INSERT INTO pending_memories "
+            "(profile_id, content, tags, metadata, created_at, status) "
+            "VALUES (?, ?, ?, ?, ?, 'pending')",
+            (profile_id, content, tags, json.dumps(metadata or {}),
+             time.strftime("%Y-%m-%dT%H:%M:%S")),
         )
         conn.commit()
         return cur.lastrowid or 0
@@ -108,20 +130,34 @@ def store_pending(
         conn.close()
 
 
-def get_pending(base_dir: Path | None = None, limit: int = 50) -> list[dict]:
-    """Get unprocessed pending memories."""
+def get_pending(
+    base_dir: Path | None = None,
+    limit: int = 50,
+    profile_id: str | None = None,
+) -> list[dict]:
+    """Get unprocessed pending memories, optionally scoped to one profile.
+
+    The drain passes ``profile_id`` = its engine's active profile so it only
+    ever claims items enqueued under that profile. Items for other profiles
+    wait until their profile is active — never materialized under the wrong one.
+    """
     conn = _get_db(base_dir)
     try:
-        rows = conn.execute(
-            "SELECT id, content, tags, metadata, created_at, retry_count "
+        query = (
+            "SELECT id, content, tags, metadata, created_at, retry_count, profile_id "
             "FROM pending_memories WHERE status = 'pending' "
-            "AND COALESCE(next_retry_at, 0) <= ? "
-            "ORDER BY id ASC LIMIT ?",
-            (time.time(), limit),
-        ).fetchall()
+            "AND COALESCE(next_retry_at, 0) <= ?"
+        )
+        params: list = [time.time()]
+        if profile_id is not None:
+            query += " AND profile_id = ?"
+            params.append(profile_id)
+        query += " ORDER BY id ASC LIMIT ?"
+        params.append(limit)
+        rows = conn.execute(query, params).fetchall()
         return [
             {"id": r[0], "content": r[1], "tags": r[2], "metadata": r[3],
-             "created_at": r[4], "retry_count": r[5]}
+             "created_at": r[4], "retry_count": r[5], "profile_id": r[6]}
             for r in rows
         ]
     finally:

package/src/superlocalmemory/cli/proxy_cmd.py

@@ -99,6 +99,10 @@ def cmd_proxy(args: Namespace) -> None:
             print("Or run: slm wrap claude")
             print()
             print("Proxy ready.")
+            print()
+            print("Note: 'slm proxy' enables the proxy independently — it does not")
+            print("flip the master optimize switch, so 'slm optimize status' may")
+            print("show OFF while the proxy is running.")
         else:
             print("Error: proxy failed to start. Check logs.", file=sys.stderr)
             sys.exit(1)

package/src/superlocalmemory/cli/scale_engine_cmd.py

@@ -37,10 +37,16 @@ def cmd_db_scale(args: Namespace) -> int:
             if not args.stage_id:
                 raise ScaleEngineError("promote requires --stage-id (see `slm db scale status`)")
             result = manager.promote(args.stage_id)
+            # The running daemon reads scale_engine_state once at startup; the
+            # promoted Cozo/Lance backends are only wired in on the next start.
+            # Without this flag users see a clean promote and keep hitting
+            # SQLite-only silently (no error, no speedup).
+            result = {**result, "restart_required": True}
         elif action == "rollback":
             if not args.backup_id:
                 raise ScaleEngineError("rollback requires --backup-id (see `slm db scale status`)")
             result = manager.rollback(args.backup_id)
+            result = {**result, "restart_required": True}
         else:
             raise ScaleEngineError(f"unknown Scale Engine action: {action}")
     except (ScaleEngineError, CanonicalVectorError) as exc:

package/src/superlocalmemory/cli/setup_wizard.py

@@ -114,17 +114,19 @@ def _download_model(model_name: str, label: str) -> bool:
     print(f"\n  Downloading {label}: {model_name}")
     print(f"  (this may take a few minutes on first run)\n")
 
+    # H-03: pass the model name as argv, never interpolated into executed
+    # source, so a crafted model_name cannot become arbitrary Python.
     script = (
-        f"import sys; "
-        f"from sentence_transformers import SentenceTransformer; "
-        f"m = SentenceTransformer('{model_name}', trust_remote_code=True); "
-        f"d = m.get_sentence_embedding_dimension(); "
-        f"print(f'OK dim={{d}}'); "
+        "import sys; "
+        "from sentence_transformers import SentenceTransformer; "
+        "m = SentenceTransformer(sys.argv[1], trust_remote_code=True); "
+        "d = m.get_sentence_embedding_dimension(); "
+        "print(f'OK dim={d}'); "
     )
 
     try:
         result = subprocess.run(
-            [sys.executable, "-c", script],
+            [sys.executable, "-c", script, model_name],
             timeout=600,  # 10 min for large model downloads
             capture_output=False,  # Show download progress
             text=True,
@@ -156,15 +158,16 @@ def _download_reranker(model_name: str) -> bool:
     print(f"\n  Downloading reranker: {model_name}")
     print(f"  (cross-encoder for result re-ranking)\n")
 
+    # H-03: model name via argv, never interpolated into executed source.
     script = (
-        f"from sentence_transformers import CrossEncoder; "
-        f"m = CrossEncoder('{model_name}', trust_remote_code=True); "
-        f"print('OK'); "
+        "import sys; from sentence_transformers import CrossEncoder; "
+        "m = CrossEncoder(sys.argv[1], trust_remote_code=True); "
+        "print('OK'); "
     )
 
     try:
         result = subprocess.run(
-            [sys.executable, "-c", script],
+            [sys.executable, "-c", script, model_name],
             timeout=300,
             capture_output=False,
             text=True,
@@ -195,16 +198,17 @@ def _download_compressor(model_name: str) -> bool:
     print(f"\n  Downloading compression model: {model_name}")
     print(f"  (LLMLingua-2 prose compressor, ~560MB — aggressive mode only)\n")
 
+    # H-03: model name via argv, never interpolated into executed source.
     script = (
-        "from llmlingua import PromptCompressor; "
-        f"PromptCompressor(model_name='{model_name}', use_llmlingua2=True, "
+        "import sys; from llmlingua import PromptCompressor; "
+        "PromptCompressor(model_name=sys.argv[1], use_llmlingua2=True, "
         "device_map='cpu'); "
         "print('OK')"
     )
 
     try:
         result = subprocess.run(
-            [sys.executable, "-c", script],
+            [sys.executable, "-c", script, model_name],
             timeout=900,  # 560MB on a slow link can exceed 5 min
             capture_output=False,
             text=True,
@@ -494,6 +498,11 @@ def run_wizard(auto: bool = False) -> None:
 
     # -- Step 4: Download models --
     print()
+    # H-06 (CVE-2025-14926): a malicious HuggingFace checkpoint can execute code
+    # at load time. SLM only downloads its pinned defaults, but warn users who
+    # point config at a custom model.
+    print("  ⚠ Only install models from sources you trust — a malicious model")
+    print("    checkpoint can run code on your machine at load time. See SECURITY.md.")
     print("─── Step 4/10: Download Embedding Model ───")
 
     if _embedding_is_remote(config):

package/src/superlocalmemory/compliance/audit.py

@@ -92,6 +92,12 @@ class AuditChain:
         conn = sqlite3.connect(path)
         conn.execute("PRAGMA journal_mode=WAL")
         conn.row_factory = sqlite3.Row
+        # C4: audit chain holds a tamper-evident record — keep it owner-only.
+        try:
+            from superlocalmemory.core.security_primitives import harden_db_perms
+            harden_db_perms(path)
+        except Exception:
+            pass
         return conn
 
     def _make_conn(self) -> sqlite3.Connection:

package/src/superlocalmemory/compliance/gdpr.py

@@ -19,6 +19,15 @@ from datetime import UTC, datetime
 
 logger = logging.getLogger(__name__)
 
+# Friendly export keys → canonical table names (stable Art.20 export contract).
+_EXPORT_ALIASES = {
+    "facts": "atomic_facts",
+    "entities": "canonical_entities",
+    "edges": "graph_edges",
+    "feedback": "feedback_records",
+    "scenes": "memory_scenes",
+}
+
 
 class GDPRCompliance:
     """GDPR compliance operations for memory data.
@@ -30,170 +39,146 @@ class GDPRCompliance:
     - Audit Trail: Log all data operations
     """
 
+    # Tables that carry a profile_id column but are NOT tenant memory to be
+    # erased/exported wholesale. `profiles` is the tenant record (handled
+    # separately, deleted last).
+    _NON_MEMORY_SCOPED = frozenset({"profiles"})
+
     def __init__(self, db) -> None:
         self._db = db
 
+    def _profile_scoped_tables(self) -> list[str]:
+        """Every table carrying a ``profile_id`` column — discovered live from
+        the schema so a newly-added table can never be silently missed by
+        export or erasure (the class of bug that breaks GDPR completeness)."""
+        try:
+            names = [
+                dict(r)["name"]
+                for r in self._db.execute(
+                    "SELECT name FROM sqlite_master WHERE type='table'"
+                )
+            ]
+        except Exception:
+            return []
+        out: list[str] = []
+        for t in names:
+            if t.startswith("sqlite_") or t in self._NON_MEMORY_SCOPED:
+                continue
+            try:
+                cols = {dict(c)["name"] for c in self._db.execute(f"PRAGMA table_info({t})")}
+            except Exception:
+                continue
+            if "profile_id" in cols:
+                out.append(t)
+        return out
+
     # -- Right to Access (Art. 15) -----------------------------------------
 
     def export_profile_data(self, profile_id: str) -> dict:
-        """Export ALL data for a profile in machine-readable format.
-
-        Returns a dict containing all memories, facts, entities,
-        edges, trust scores, feedback, and behavioral patterns.
-        """
+        """Export ALL data for a profile in machine-readable format (Art. 15 /
+        Art. 20). Covers every profile-scoped table discovered from the schema,
+        plus the profile record itself."""
         self._audit("export", "profile", profile_id, "Full data export")
 
         data: dict = {"profile_id": profile_id, "exported_at": _now()}
+        for table in self._profile_scoped_tables():
+            try:
+                rows = self._db.execute(
+                    f"SELECT * FROM {table} WHERE profile_id = ?", (profile_id,)
+                )
+                data[table] = [dict(r) for r in rows]
+            except Exception as exc:  # pragma: no cover — defensive per-table
+                logger.warning("export: table %s skipped: %s", table, exc)
+
+        # Profile record itself (the tenant metadata).
+        try:
+            rows = self._db.execute(
+                "SELECT * FROM profiles WHERE profile_id = ?", (profile_id,)
+            )
+            data["profile_record"] = [dict(r) for r in rows]
+        except Exception:
+            data["profile_record"] = []
 
-        # Memories
-        rows = self._db.execute(
-            "SELECT * FROM memories WHERE profile_id = ?", (profile_id,)
-        )
-        data["memories"] = [dict(r) for r in rows]
-
-        # Facts
-        rows = self._db.execute(
-            "SELECT * FROM atomic_facts WHERE profile_id = ?", (profile_id,)
-        )
-        data["facts"] = [dict(r) for r in rows]
-
-        # Entities
-        rows = self._db.execute(
-            "SELECT * FROM canonical_entities WHERE profile_id = ?", (profile_id,)
-        )
-        data["entities"] = [dict(r) for r in rows]
-
-        # Graph edges
-        rows = self._db.execute(
-            "SELECT * FROM graph_edges WHERE profile_id = ?", (profile_id,)
-        )
-        data["edges"] = [dict(r) for r in rows]
-
-        # Trust scores
-        rows = self._db.execute(
-            "SELECT * FROM trust_scores WHERE profile_id = ?", (profile_id,)
-        )
-        data["trust_scores"] = [dict(r) for r in rows]
-
-        # Feedback
-        rows = self._db.execute(
-            "SELECT * FROM feedback_records WHERE profile_id = ?", (profile_id,)
-        )
-        data["feedback"] = [dict(r) for r in rows]
-
-        # Entity profiles
-        rows = self._db.execute(
-            "SELECT * FROM entity_profiles WHERE profile_id = ?", (profile_id,)
-        )
-        data["entity_profiles"] = [dict(r) for r in rows]
-
-        # Memory scenes
-        rows = self._db.execute(
-            "SELECT * FROM memory_scenes WHERE profile_id = ?", (profile_id,)
-        )
-        data["scenes"] = [dict(r) for r in rows]
-
-        # Temporal events
-        rows = self._db.execute(
-            "SELECT * FROM temporal_events WHERE profile_id = ?", (profile_id,)
-        )
-        data["temporal_events"] = [dict(r) for r in rows]
-
-        # Consolidation log
-        rows = self._db.execute(
-            "SELECT * FROM consolidation_log WHERE profile_id = ?", (profile_id,)
-        )
-        data["consolidation_log"] = [dict(r) for r in rows]
-
-        # Behavioral patterns
-        rows = self._db.execute(
-            "SELECT * FROM behavioral_patterns WHERE profile_id = ?", (profile_id,)
-        )
-        data["behavioral_patterns"] = [dict(r) for r in rows]
-
-        # Action outcomes
-        rows = self._db.execute(
-            "SELECT * FROM action_outcomes WHERE profile_id = ?", (profile_id,)
-        )
-        data["action_outcomes"] = [dict(r) for r in rows]
-
-        # Compliance audit trail
-        rows = self._db.execute(
-            "SELECT * FROM compliance_audit WHERE profile_id = ?", (profile_id,)
-        )
-        data["compliance_audit"] = [dict(r) for r in rows]
-
-        # Provenance (data lineage — EU AI Act Art. 10)
-        rows = self._db.execute(
-            "SELECT * FROM provenance WHERE profile_id = ?", (profile_id,)
-        )
-        data["provenance"] = [dict(r) for r in rows]
-
-        # Entity aliases (indirect PII via entity relationships)
-        rows = self._db.execute(
-            "SELECT ea.* FROM entity_aliases ea "
-            "JOIN canonical_entities ce ON ea.entity_id = ce.entity_id "
-            "WHERE ce.profile_id = ?", (profile_id,)
-        )
-        data["entity_aliases"] = [dict(r) for r in rows]
-
-        # Profile record itself
-        rows = self._db.execute(
-            "SELECT * FROM profiles WHERE profile_id = ?", (profile_id,)
-        )
-        data["profile_record"] = [dict(r) for r in rows]
-
+        # total_items counts the canonical (table-name) keys only, before
+        # friendly aliases are added, so it is not double-counted.
         data["total_items"] = sum(
             len(v) for v in data.values() if isinstance(v, list)
         )
 
+        # Backward-compatible friendly aliases for the well-known keys (stable
+        # export contract) — they reference the same lists, not copies.
+        for friendly, table in _EXPORT_ALIASES.items():
+            if table in data:
+                data[friendly] = data[table]
+
         logger.info("Exported %d items for profile '%s'", data["total_items"], profile_id)
         return data
 
     # -- Right to Erasure (Art. 17) ----------------------------------------
 
     def forget_profile(self, profile_id: str) -> dict:
-        """Delete ALL data for a profile (right to be forgotten).
+        """Delete ALL data for a profile (right to be forgotten, Art. 17).
 
-        CASCADE deletes handle most cleanup via foreign keys.
-        Returns counts of deleted items.
+        Erases every profile-scoped table discovered from the live schema, so a
+        newly-added table is covered automatically. The erasure is recorded in
+        the tamper-proof audit chain BEFORE any deletion (Art. 5(2)
+        accountability) — the in-DB compliance_audit row is itself erased, so
+        the chain in a separate DB is the durable evidence.
         """
         if profile_id == "default":
             raise ValueError("Cannot delete the default profile via GDPR erasure. "
                              "Use profile deletion instead.")
 
+        # 1) Durable, tamper-evident record FIRST — survives the erasure.
+        try:
+            from superlocalmemory.compliance.audit import AuditChain
+            from superlocalmemory.infra.data_root import state_path
+            AuditChain(str(state_path("audit_chain.db"))).log(
+                "gdpr_erase", agent_id="gdpr", profile_id=profile_id,
+                metadata={"basis": "GDPR Art.17 right-to-erasure"},
+            )
+        except Exception as exc:
+            logger.warning("GDPR erase: audit-chain log failed: %s", exc)
         self._audit("delete", "profile", profile_id, "GDPR erasure request")
 
         counts: dict[str, int] = {}
-        tables = [
-            "compliance_audit", "action_outcomes", "behavioral_patterns",
-            "feedback_records", "trust_scores", "provenance",
-            "consolidation_log", "graph_edges", "temporal_events",
-            "memory_scenes", "entity_profiles", "bm25_tokens",
-            "atomic_facts", "memories", "canonical_entities",
-        ]
+        tables = self._profile_scoped_tables()
+        # Pass 1 — count every table BEFORE any deletion, so a CASCADE that
+        # removes a child (e.g. atomic_facts via memories) does not zero the
+        # attribution. Completeness is independent of this.
         for table in tables:
-            rows = self._db.execute(
-                f"SELECT COUNT(*) AS c FROM {table} WHERE profile_id = ?",
-                (profile_id,),
-            )
-            counts[table] = int(dict(rows[0])["c"]) if rows else 0
-            self._db.execute(
-                f"DELETE FROM {table} WHERE profile_id = ?", (profile_id,)
-            )
-
-        # Delete entity aliases (orphan PII via entity relationships)
-        self._db.execute(
-            "DELETE FROM entity_aliases WHERE entity_id IN "
-            "(SELECT entity_id FROM canonical_entities WHERE profile_id = ?)",
-            (profile_id,),
-        )
-
-        # Delete profile itself
-        self._db.execute(
-            "DELETE FROM profiles WHERE profile_id = ?", (profile_id,)
-        )
-        counts["profiles"] = 1
+            try:
+                rows = self._db.execute(
+                    f"SELECT COUNT(*) AS c FROM {table} WHERE profile_id = ?",
+                    (profile_id,),
+                )
+                counts[table] = int(dict(rows[0])["c"]) if rows else 0
+            except Exception as exc:  # pragma: no cover
+                logger.warning("GDPR erase: count %s failed: %s", table, exc)
+                counts[table] = 0
+        # Pass 2 — full-tenant wipe with FK enforcement OFF so table order is
+        # irrelevant (every profile row in every table goes). FTS shadow rows
+        # are still removed by the base-table delete triggers.
+        try:
+            self._db.execute("PRAGMA foreign_keys=OFF")
+        except Exception:
+            pass
+        try:
+            for table in tables:
+                try:
+                    self._db.execute(
+                        f"DELETE FROM {table} WHERE profile_id = ?", (profile_id,)
+                    )
+                except Exception as exc:  # pragma: no cover — defensive per-table
+                    logger.warning("GDPR erase: delete %s failed: %s", table, exc)
+            # Delete the profile record itself.
+            self._db.execute("DELETE FROM profiles WHERE profile_id = ?", (profile_id,))
+            counts["profiles"] = 1
+        finally:
+            try:
+                self._db.execute("PRAGMA foreign_keys=ON")
+            except Exception:
+                pass
 
         # Erase learning database (separate DB file)
         try:
@@ -211,7 +196,7 @@ class GDPRCompliance:
         except Exception:
             pass
 
-        logger.info("GDPR erasure for '%s': %s", profile_id, counts)
+        logger.info("GDPR erasure for '%s': %d tables, %s", profile_id, len(tables), counts)
         return counts
 
     def forget_entity(self, entity_name: str, profile_id: str) -> dict:
@@ -254,9 +239,14 @@ class GDPRCompliance:
             (eid, profile_id),
         )
 
-        # Delete aliases + entity
-        self._db.execute("DELETE FROM entity_aliases WHERE entity_id = ?", (eid,))
-        self._db.execute("DELETE FROM canonical_entities WHERE entity_id = ?", (eid,))
+        # Delete aliases + entity (profile-scoped — entity_id is UUID-global but
+        # keep the tenant predicate for consistent Art.17 isolation).
+        self._db.execute(
+            "DELETE FROM entity_aliases WHERE entity_id = ? AND profile_id = ?",
+            (eid, profile_id))
+        self._db.execute(
+            "DELETE FROM canonical_entities WHERE entity_id = ? AND profile_id = ?",
+            (eid, profile_id))
         counts["entity"] = 1
 
         logger.info("Entity erasure '%s' in '%s': %s", entity_name, profile_id, counts)