superlocalmemory 3.7.8 → 3.8.1

package/src/superlocalmemory/core/pii.py

@@ -0,0 +1,105 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under AGPL-3.0-or-later - see LICENSE file
+# Part of SuperLocalMemory V3 — PII redaction on ingest (C4)
+
+"""Opt-in PII redaction for ingested memory content.
+
+For team / company deployments an operator may need memory to never persist
+personal identifiers (email, phone, national ID, payment card, IP). This
+module provides a pure, well-bounded scrubber that replaces detected PII with
+``[PII:TYPE]`` markers. It is complementary to ``security_primitives.
+redact_secrets`` (which handles API keys / tokens and always runs).
+
+Design goals:
+* **Low false-positive rate.** Card numbers are Luhn-validated; SSNs use the
+  canonical grouping; phone matching requires a plausible separator shape.
+* **Deterministic + pure.** No I/O, no config — the caller decides when to run
+  it (gated by SLM_PII_REDACTION / config), so it is trivially testable.
+* **Order matters.** Emails are redacted before phone/number sweeps so an
+  email's local part is never mistaken for a number.
+"""
+
+from __future__ import annotations
+
+import re
+
+# Order-sensitive: earlier patterns win over later ones on overlapping spans.
+_EMAIL = re.compile(r"\b[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}\b")
+_SSN = re.compile(r"\b\d{3}-\d{2}-\d{4}\b")
+_IPV4 = re.compile(
+    r"\b(?:(?:25[0-5]|2[0-4]\d|1?\d?\d)\.){3}(?:25[0-5]|2[0-4]\d|1?\d?\d)\b"
+)
+# Phone: conservative to avoid eating ISO dates (4-2-2) / version strings.
+# Only unambiguous shapes match:
+#   * international  +CC then grouped digits
+#   * parenthesized  (415) 555-0132
+#   * strict US      415-555-0132 / 415.555.0132  (3-3-4, dot/dash only — a
+#     space separator is excluded so "2026-07-22 12" style runs never match).
+_PHONE = re.compile(
+    r"(?<!\w)(?:"
+    r"\+\d{1,3}[\s.\-]?\d{1,4}[\s.\-]?\d{2,4}[\s.\-]?\d{2,4}"
+    r"|\(\d{3}\)[\s.\-]?\d{3}[\s.\-]?\d{4}"
+    r"|\d{3}[.\-]\d{3}[.\-]\d{4}"
+    r")(?!\w)"
+)
+# Candidate card: 13–19 digits, optionally grouped by space/dash. Luhn-checked.
+_CARD_CANDIDATE = re.compile(r"(?<!\w)(?:\d[ -]?){13,19}(?!\w)")
+
+
+def _luhn_ok(digits: str) -> bool:
+    """Return True if ``digits`` (0-9 only) passes the Luhn checksum."""
+    if not (13 <= len(digits) <= 19):
+        return False
+    total = 0
+    parity = len(digits) % 2
+    for i, ch in enumerate(digits):
+        d = ord(ch) - 48
+        if i % 2 == parity:
+            d *= 2
+            if d > 9:
+                d -= 9
+        total += d
+    return total % 10 == 0
+
+
+def _redact_cards(text: str, counter: list[int]) -> str:
+    def _sub(m: re.Match[str]) -> str:
+        raw = m.group(0)
+        digits = re.sub(r"\D", "", raw)
+        if _luhn_ok(digits):
+            counter[0] += 1
+            return "[PII:CARD]"
+        return raw
+    return _CARD_CANDIDATE.sub(_sub, text)
+
+
+def redact_pii(text: str) -> tuple[str, int]:
+    """Return ``(redacted_text, num_redactions)``.
+
+    Never raises; a non-string or empty input is returned unchanged with 0.
+    """
+    if not isinstance(text, str) or not text:
+        return text, 0
+
+    counter = [0]
+
+    def _count_sub(pattern: re.Pattern[str], label: str, s: str) -> str:
+        def _sub(_m: re.Match[str]) -> str:
+            counter[0] += 1
+            return label
+        return pattern.sub(_sub, s)
+
+    out = text
+    # Email first (protects local parts from the number sweeps).
+    out = _count_sub(_EMAIL, "[PII:EMAIL]", out)
+    # Payment cards before generic phone/number matching (Luhn-gated).
+    out = _redact_cards(out, counter)
+    out = _count_sub(_SSN, "[PII:SSN]", out)
+    out = _count_sub(_IPV4, "[PII:IP]", out)
+    out = _count_sub(_PHONE, "[PII:PHONE]", out)
+    return out, counter[0]
+
+
+def redact_pii_text(text: str) -> str:
+    """Convenience wrapper returning only the redacted string."""
+    return redact_pii(text)[0]

package/src/superlocalmemory/core/progressive_abstraction.py

@@ -0,0 +1,208 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under AGPL-3.0-or-later - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""Progressive abstraction (Wave Q3) — the top persona tier + drill-down.
+
+Completes the abstraction hierarchy on the ONE principled backbone (rather
+than a fourth divergent clustering):
+
+    atoms (atomic_facts)
+      -> entity communities        (Wave Q backbone)
+        -> community summaries      (Wave Q2)
+          -> persona roll-up        (this module)
+
+The persona is one bounded roll-up per profile that consumes the top community
+summaries. It is recall-GATED (never auto-injected into the hot recall path —
+avoids the V3.4.40 summary-pollution regression) and SIZE-bounded. Drill-down
+(``get_sources``) walks the hierarchy back down to the source atoms, matching
+the market bar for summary->source provenance (Zep-style).
+
+Runs in the background consolidation lane after community summaries.
+Fail-open throughout; recompute replaces a profile's row.
+
+Part of Qualixar | Author: Varun Pratap Bhardwaj
+License: AGPL-3.0-or-later
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from typing import Any
+
+from superlocalmemory.core.community_summary import CommunitySummaryBuilder
+
+logger = logging.getLogger(__name__)
+
+_PERSONA_MAX_CHARS = 2048
+
+
+class ProgressiveAbstraction:
+    """Build + persist the persona tier; provide hierarchy drill-down."""
+
+    def __init__(
+        self,
+        db: Any,
+        summarizer: Any = None,
+        max_communities_in_persona: int = 8,
+        persona_max_chars: int = _PERSONA_MAX_CHARS,
+        max_keywords: int = 12,
+    ) -> None:
+        self._db = db
+        self._summarizer = summarizer
+        self._max_communities = max(1, int(max_communities_in_persona))
+        self._persona_max_chars = max(256, int(persona_max_chars))
+        self._max_keywords = max(1, int(max_keywords))
+
+    # ------------------------------------------------------------------
+    # Build
+    # ------------------------------------------------------------------
+
+    def compute_and_store(self, profile_id: str) -> dict[str, Any]:
+        summaries = CommunitySummaryBuilder(self._db).get_summaries(profile_id)
+        try:
+            self._db.execute(
+                "DELETE FROM persona_summary WHERE profile_id = ?",
+                (profile_id,),
+            )
+        except Exception as exc:
+            logger.debug("persona_summary clear failed: %s", exc)
+        if not summaries:
+            return {"built": False, "communities_in_persona": 0}
+
+        top = summaries[: self._max_communities]
+        summary = self._persona_summary(top)
+        keywords = self._merge_keywords(top)
+        community_ids = [int(s["community_id"]) for s in top]
+
+        try:
+            self._db.execute(
+                "INSERT OR REPLACE INTO persona_summary "
+                "(profile_id, summary, keywords, community_ids_json, computed_at) "
+                "VALUES (?, ?, ?, ?, datetime('now'))",
+                (profile_id, summary, keywords, json.dumps(community_ids)),
+            )
+        except Exception as exc:
+            logger.debug("persona_summary write failed: %s", exc)
+            return {"built": False, "communities_in_persona": 0}
+
+        return {"built": True, "communities_in_persona": len(top)}
+
+    # ------------------------------------------------------------------
+    # Read API
+    # ------------------------------------------------------------------
+
+    def get_persona(self, profile_id: str) -> dict | None:
+        try:
+            rows = self._db.execute(
+                "SELECT * FROM persona_summary WHERE profile_id = ?",
+                (profile_id,),
+            )
+        except Exception as exc:
+            logger.debug("get_persona failed: %s", exc)
+            return None
+        if not rows:
+            return None
+        d = dict(rows[0])
+        try:
+            community_ids = json.loads(d.get("community_ids_json") or "[]")
+        except (ValueError, TypeError):
+            community_ids = []
+        return {
+            "profile_id": d.get("profile_id", profile_id),
+            "summary": d.get("summary", ""),
+            "keywords": d.get("keywords", ""),
+            "community_ids": community_ids,
+            "computed_at": d.get("computed_at", ""),
+        }
+
+    def get_sources(self, profile_id: str, node_id: Any) -> dict:
+        """Drill-down: a tier node -> its child communities + source atoms.
+
+        node_id == "persona"  -> the persona's member communities + their facts.
+        node_id == <community> -> that community's member facts.
+        Unknown node -> empty (never raises).
+        """
+        result: dict[str, Any] = {
+            "node_id": node_id, "node_type": "unknown",
+            "communities": [], "fact_ids": [],
+        }
+        try:
+            if isinstance(node_id, str) and node_id.lower() == "persona":
+                persona = self.get_persona(profile_id)
+                cids = persona["community_ids"] if persona else []
+                fact_ids: list[str] = []
+                seen: set[str] = set()
+                for cid in cids:
+                    for fid in self._community_fact_ids(profile_id, cid):
+                        if fid not in seen:
+                            seen.add(fid)
+                            fact_ids.append(fid)
+                result.update(
+                    node_type="persona", communities=list(cids), fact_ids=fact_ids,
+                )
+                return result
+
+            # Otherwise treat node_id as a community id.
+            cid = int(node_id)
+            fids = self._community_fact_ids(profile_id, cid)
+            result.update(
+                node_type="community", communities=[cid], fact_ids=fids,
+            )
+            return result
+        except (ValueError, TypeError):
+            return result
+        except Exception as exc:
+            logger.debug("get_sources failed: %s", exc)
+            return result
+
+    # ------------------------------------------------------------------
+    # Internal
+    # ------------------------------------------------------------------
+
+    def _community_fact_ids(self, profile_id: str, community_id: Any) -> list[str]:
+        try:
+            rows = self._db.execute(
+                "SELECT fact_ids_json FROM community_summaries "
+                "WHERE profile_id = ? AND community_id = ?",
+                (profile_id, int(community_id)),
+            )
+        except Exception as exc:
+            logger.debug("_community_fact_ids failed: %s", exc)
+            return []
+        if not rows:
+            return []
+        try:
+            return [str(f) for f in json.loads(dict(rows[0]).get("fact_ids_json") or "[]")]
+        except (ValueError, TypeError):
+            return []
+
+    def _persona_summary(self, top: list[dict]) -> str:
+        if self._summarizer is not None:
+            try:
+                text = self._summarizer.summarize_cluster(
+                    [{"content": s.get("summary", "")} for s in top],
+                )
+                if text and text.strip():
+                    return text.strip()[: self._persona_max_chars]
+            except Exception as exc:
+                logger.debug("persona summarizer failed (fail-open): %s", exc)
+        # Mode A keyword-dense fallback: stitch the top community summaries.
+        heads = [s.get("summary", "").strip() for s in top if s.get("summary")]
+        base = " ".join(heads) if heads else "No persona yet."
+        return base[: self._persona_max_chars]
+
+    def _merge_keywords(self, top: list[dict]) -> str:
+        seen: set[str] = set()
+        merged: list[str] = []
+        for s in top:
+            for kw in (s.get("keywords", "") or "").split(","):
+                k = kw.strip()
+                low = k.lower()
+                if k and low not in seen:
+                    seen.add(low)
+                    merged.append(k)
+                    if len(merged) >= self._max_keywords:
+                        return ", ".join(merged)
+        return ", ".join(merged)

package/src/superlocalmemory/core/recall_pipeline.py

@@ -623,6 +623,7 @@ def run_recall(
     fast: bool = False,
     include_global: bool = False,
     include_shared: bool = False,
+    window: str | tuple[str, str] | None = None,
 ) -> RecallResponse:
     """Recall relevant facts for a query.
 
@@ -631,8 +632,8 @@ def run_recall(
 
     Pipeline: retrieval -> agentic sufficiency (if configured) -> post-recall updates.
 
-    ``fast=True`` skips spreading activation and remote agentic verification,
-    retaining the bounded single-pass retrieval channels.
+    ``fast=True`` skips remote agentic verification while retaining the six
+    local retrieval channels.
     """
     # Pre-operation hooks
     hook_ctx = {
@@ -657,12 +658,15 @@ def run_recall(
             logger.warning("[RECALL-TIMING] %-22s %.0f ms",
                            _label, (_time_t.monotonic() - _t0) * 1000.0)
 
-    extra_disabled = {"spreading_activation"} if fast else None
+    # The interactive path must retain the complete local retrieval contract.
+    # Only agentic verification can invoke an unbounded model round.
+    extra_disabled = None
     response = retrieval_engine.recall(
         query, profile_id, m, limit,
         extra_disabled_channels=extra_disabled,
         include_global=include_global,
         include_shared=include_shared,
+        window=window,
     )
     _mark("retrieval(chan+rerank)")
 

package/src/superlocalmemory/core/recall_worker.py

@@ -62,6 +62,7 @@ def _get_engine():
 def _handle_recall(
     query: str, limit: int, session_id: str = "", fast: bool = False,
     include_global: bool | None = None, include_shared: bool | None = None,
+    window: str | None = None,
 ) -> dict:
     engine = _get_engine()
     # v3.6.15 multi-scope: None flags let engine.recall resolve the configured
@@ -70,11 +71,19 @@ def _handle_recall(
     response = engine.recall(
         query, limit=limit, session_id=session_id or None, fast=bool(fast),
         include_global=include_global, include_shared=include_shared,
+        window=window or None,
     )
 
-    # Batch-fetch original memory text for all results
+    # Batch-fetch original memory text for all results. Retrieval already
+    # enforced scope, so resolve content for everything it returned (own +
+    # global + shared-with-me); another tenant's PRIVATE content still can't
+    # resolve. Using the raw (possibly-None) recall flags here would drop
+    # content for legitimately-recalled global/shared memories.
     memory_ids = list({r.fact.memory_id for r in response.results[:limit] if r.fact.memory_id})
-    memory_map = engine._db.get_memory_content_batch(memory_ids) if memory_ids else {}
+    memory_map = engine._db.get_memory_content_batch(
+        memory_ids, engine.profile_id,
+        include_global=True, include_shared=True,
+    ) if memory_ids else {}
 
     # v3.6.6: same shared chokepoint as the daemon HTTP route + CLI fallback,
     # so the MCP WorkerPool subprocess path returns identical budgeted output.
@@ -172,11 +181,15 @@ def _handle_store(content: str, metadata: dict) -> dict:
 def _handle_get_memory_facts(memory_id: str) -> dict:
     engine = _get_engine()
     pid = engine.profile_id
-    # Get original memory content
-    mem_map = engine._db.get_memory_content_batch([memory_id])
+    # Get original memory content (C4: tenant-scoped; global/shared resolvable)
+    mem_map = engine._db.get_memory_content_batch(
+        [memory_id], pid, include_global=True, include_shared=True,
+    )
     original = mem_map.get(memory_id, "")
-    # Get child facts
-    facts = engine._db.get_facts_by_memory_id(memory_id, pid)
+    # Get child facts — same scope as the content fetch so a shared/global
+    # memory's facts are not silently empty.
+    facts = engine._db.get_facts_by_memory_id(
+        memory_id, pid, include_global=True, include_shared=True)
     fact_list = []
     for f in facts:
         fact_list.append({
@@ -308,6 +321,7 @@ def _worker_main() -> None:
                     req.get("session_id", ""), bool(req.get("fast", False)),
                     include_global=req.get("include_global"),
                     include_shared=req.get("include_shared"),
+                    window=req.get("window"),
                 )
                 _respond(result)
             elif cmd == "store":

package/src/superlocalmemory/core/scale_engine.py

@@ -116,6 +116,13 @@ class ScaleEngineManager:
             # This command reads persisted state, not the live daemon. Never
             # turn a last-known backend row into a present-tense routing claim.
             "active": {"cozo": False, "lance": False},
+            # LOW-2 (3.7.9): after promote the daemon must restart before the
+            # backends actually serve; surface that explicitly so `status` isn't
+            # mistaken for "promotion failed".
+            "daemon_must_restart_to_activate": (
+                state == "promoted"
+                and not any(v == "active" for v in runtime.values())
+            ),
             "last_daemon_observation": runtime,
             "paths_present": paths_present,
             "retrieval_routing": (
@@ -319,6 +326,8 @@ class ScaleEngineManager:
                 raise ScaleEngineError(
                     f"projection parity failed: canonical={canonical}, observed={observed}"
                 )
+            with self._readonly_connection() as conn:
+                self._verify_content_sample(conn, cozo, canonical)
             manifest.update({"state": "verified", "verified_at": _utc_now(), "observed": observed})
             self._write_manifest(stage_dir, manifest)
             self.config.scale_engine_state = "verified"
@@ -510,7 +519,31 @@ class ScaleEngineManager:
         ).fetchone()[0]
         edges = count_logical_edges(conn, self.profile_id)
         vectors = count_canonical_vectors(conn, self.profile_id)
-        return {"entities": int(nodes), "edges": int(edges), "vectors": int(vectors)}
+        fact_entity = self._count_fact_entity_links(conn)
+        return {
+            "entities": int(nodes),
+            "edges": int(edges),
+            "vectors": int(vectors),
+            "fact_entity": int(fact_entity),
+        }
+
+    def _count_fact_entity_links(self, conn: sqlite3.Connection) -> int:
+        """Count fact->entity bridge rows exactly as ``bulk_import_from_sqlite``
+        projects them: dedup entity IDs per fact, drop empties. This bridge is
+        what lets Cozo map a query seed into the fact graph; if its import
+        silently fails, count parity on entities/edges/vectors still passes but
+        entity recall returns empty — so it must be verified explicitly."""
+        total = 0
+        for (raw,) in conn.execute(
+            "SELECT canonical_entities_json FROM atomic_facts WHERE profile_id=?",
+            (self.profile_id,),
+        ):
+            try:
+                entity_ids = json.loads(raw or "[]")
+            except (TypeError, ValueError, json.JSONDecodeError):
+                continue
+            total += sum(1 for eid in dict.fromkeys(entity_ids) if eid)
+        return total
 
     def _observed_counts(self, cozo: Any, lance: Any) -> dict[str, int]:
         graph = cozo.health_check()
@@ -521,8 +554,34 @@ class ScaleEngineManager:
             "entities": int(graph["entities"]),
             "edges": int(graph["edges"]),
             "vectors": int(vector["vectors"]),
+            "fact_entity": int(graph.get("fact_entity", 0)),
         }
 
+    def _verify_content_sample(
+        self, conn: sqlite3.Connection, cozo: Any, canonical: dict[str, int]
+    ) -> None:
+        """Content parity: count equality does not prove the import reproduced
+        the correct rows. Compare the projected entity-ID set against canonical
+        SQLite (bounded), catching a projection that has the right entity count
+        but the wrong identities."""
+        getter = getattr(cozo, "entity_ids", None)
+        n = int(canonical.get("entities", 0))
+        if not callable(getter) or n == 0:
+            return  # backend cannot sample, or nothing to compare
+        observed_ids = set(getter(limit=n))
+        rows = conn.execute(
+            "SELECT entity_id FROM canonical_entities WHERE profile_id=?",
+            (self.profile_id,),
+        ).fetchall()
+        expected_ids = {r[0] for r in rows}
+        if expected_ids != observed_ids:
+            missing = sorted(expected_ids - observed_ids)[:5]
+            extra = sorted(observed_ids - expected_ids)[:5]
+            raise ScaleEngineError(
+                "projection content parity failed: entity IDs diverged "
+                f"(missing sample={missing}, unexpected sample={extra})"
+            )
+
     def _projection_fingerprint(
         self, conn: sqlite3.Connection, counts: dict[str, int]
     ) -> str:

package/src/superlocalmemory/core/security_primitives.py

@@ -399,6 +399,35 @@ def redact_secrets(text: str, *, entropy_threshold: float = 4.5,
 # ---------------------------------------------------------------------------
 
 
+def harden_db_perms(db_path: str | Path) -> None:
+    """Restrict a database file (and its WAL/SHM sidecars) to owner-only 0600
+    and its parent directory to 0700 (C4 encryption-at-rest defense-in-depth).
+
+    Best-effort and POSIX-only: on Windows or if the file does not exist yet
+    this is a silent no-op. The daemon's data dir is already 0700, but the DB
+    files themselves shipped 0644 (world-readable); on a shared host that is a
+    real exposure even with full-disk encryption at rest.
+    """
+    if _is_windows():
+        return
+    try:
+        p = Path(db_path)
+        parent = p.parent
+        try:
+            os.chmod(parent, 0o700)
+        except OSError:
+            pass
+        for suffix in ("", "-wal", "-shm"):
+            f = Path(str(p) + suffix)
+            if f.exists():
+                try:
+                    os.chmod(f, 0o600)
+                except OSError:
+                    pass
+    except Exception:  # pragma: no cover — never block DB open on a chmod
+        pass
+
+
 def _install_token_path() -> Path:  # pragma: no cover — monkeypatched in tests
     """Default install-token location — override in tests via monkeypatch."""
     from superlocalmemory.infra.data_root import state_path
@@ -598,8 +627,17 @@ def run_subprocess_safe(
       - Restricted environment by default — only a minimal set of safe keys.
       - Callers may pass an explicit ``env`` to add specific variables.
 
-    This is the ONE place in the codebase allowed to call ``subprocess.run``.
-    Grep guard in CI enforces this (LLD-07 §7 SEC-HR-06).
+    This is the PREFERRED wrapper for any subprocess whose ``argv`` includes
+    dynamic or externally-influenced values — routing them through here keeps
+    ``shell=False``, a mandatory timeout, and a restricted environment.
+
+    It is not the *only* ``subprocess.run`` call site: a small set of vetted
+    callers invoke ``subprocess.run`` directly where they need inherited stdio
+    for live progress, a long-lived managed process, or process-group control
+    (e.g. model downloads in ``cli/setup_wizard.py``, ``infra/process_reaper``,
+    ``cli/service_installer``). Those pass only fixed/argv-quoted values — never
+    a shell string. Do not read this wrapper as a guarantee that no other
+    subprocess call exists; audit new call sites individually. (L-01, 3.7.9)
     """
     if not isinstance(argv, list):
         raise TypeError("argv must be list[str], shell=False only")