superlocalmemory 3.7.8 → 3.8.0

package/src/superlocalmemory/core/community_summary.py

@@ -0,0 +1,267 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under AGPL-3.0-or-later - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""Community summaries (Wave Q2) — one synthesized report per entity community.
+
+Rides on the entity-community backbone (core.entity_community). For each
+community it gathers the member entities' facts, EXCLUDES superseded facts
+(bi-temporal — market CRIT-3), then produces:
+
+  - a keyword-dense signal (always, Mode A, zero-LLM), and
+  - a summary: Mode B/C LLM synthesis via the shared core.Summarizer
+    (which itself falls back to a heuristic), else the Mode A keyword-dense
+    line. Fail-open — a summarizer error never breaks generation.
+
+Surfacing is on-device-safe: summaries are PRECOMPUTED here in the background
+and later matched to a query as a single thematic-context block (Q2b) — never
+a GraphRAG-style per-query LLM fan-out (market CRIT-1). member_fact_ids gives
+drill-down back to the source atoms (Q3).
+
+Part of Qualixar | Author: Varun Pratap Bhardwaj
+License: AGPL-3.0-or-later
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from collections import Counter, defaultdict
+from typing import Any
+
+from superlocalmemory.core.entity_community import EntityCommunityBuilder
+
+logger = logging.getLogger(__name__)
+
+_STOPWORDS = frozenset({
+    "the", "a", "an", "is", "was", "were", "are", "be", "been", "being",
+    "have", "has", "had", "do", "does", "did", "will", "would", "could",
+    "should", "may", "might", "shall", "can", "to", "of", "in", "for", "on",
+    "with", "at", "by", "from", "as", "into", "through", "and", "but", "or",
+    "not", "no", "this", "that", "these", "those", "it", "its", "they",
+    "them", "their", "he", "she", "his", "her", "we", "our", "you", "your",
+    "i", "my", "me", "his", "was", "who", "what", "when", "where", "how",
+})
+
+
+class CommunitySummaryBuilder:
+    """Generate + persist one summary per entity community (background)."""
+
+    def __init__(
+        self,
+        db: Any,
+        summarizer: Any = None,
+        max_communities: int = 50,
+        min_facts: int = 2,
+        max_facts_per_community: int = 30,
+        max_keywords: int = 8,
+        summary_max_chars: int = 512,
+    ) -> None:
+        self._db = db
+        self._summarizer = summarizer
+        self._max_communities = max(1, int(max_communities))
+        self._min_facts = max(1, int(min_facts))
+        self._max_facts = max(1, int(max_facts_per_community))
+        self._max_keywords = max(1, int(max_keywords))
+        self._summary_max_chars = max(64, int(summary_max_chars))
+
+    # ------------------------------------------------------------------
+    # Generation
+    # ------------------------------------------------------------------
+
+    def compute_and_store(self, profile_id: str) -> dict[str, int]:
+        communities = EntityCommunityBuilder(self._db).get_communities(profile_id)
+        try:
+            self._db.execute(
+                "DELETE FROM community_summaries WHERE profile_id = ?",
+                (profile_id,),
+            )
+        except Exception as exc:
+            logger.debug("community_summaries clear failed: %s", exc)
+        if not communities:
+            return {"summaries_written": 0, "communities": 0}
+
+        entity_to_cid: dict[str, int] = {
+            e: cid for cid, ents in communities.items() for e in ents
+        }
+        cid_facts = self._gather_facts(profile_id, entity_to_cid)
+        name_map = self._entity_names(profile_id, communities)
+
+        written = 0
+        ordered = sorted(
+            cid_facts.items(), key=lambda kv: len(kv[1]), reverse=True,
+        )
+        for cid, facts in ordered:
+            if written >= self._max_communities:
+                break
+            seen: set[str] = set()
+            vf: list[tuple[str, str]] = []
+            for fid, content in facts:
+                if fid in seen:
+                    continue
+                seen.add(fid)
+                vf.append((fid, content))
+            if len(vf) < self._min_facts:
+                continue
+
+            fact_ids = [fid for fid, _ in vf]
+            contents = [c for _, c in vf][: self._max_facts]
+            entity_ids = list(communities.get(cid, []))
+            entity_names = [name_map.get(e, e) for e in entity_ids]
+            keywords = self._keywords(contents)
+            summary = self._summary(contents, entity_names, keywords)
+
+            try:
+                self._db.execute(
+                    "INSERT OR REPLACE INTO community_summaries "
+                    "(profile_id, community_id, summary, keywords, "
+                    " entity_ids_json, fact_ids_json, fact_count, computed_at) "
+                    "VALUES (?, ?, ?, ?, ?, ?, ?, datetime('now'))",
+                    (
+                        profile_id, cid, summary, keywords,
+                        json.dumps(entity_ids), json.dumps(fact_ids),
+                        len(fact_ids),
+                    ),
+                )
+                written += 1
+            except Exception as exc:
+                logger.debug("community_summaries write failed (%s): %s", cid, exc)
+
+        return {"summaries_written": written, "communities": len(communities)}
+
+    # ------------------------------------------------------------------
+    # Read API
+    # ------------------------------------------------------------------
+
+    def get_summaries(self, profile_id: str) -> list[dict]:
+        try:
+            rows = self._db.execute(
+                "SELECT * FROM community_summaries WHERE profile_id = ? "
+                "ORDER BY fact_count DESC",
+                (profile_id,),
+            )
+        except Exception as exc:
+            logger.debug("get_summaries failed: %s", exc)
+            return []
+        return [dict(r) for r in rows]
+
+    def get_summary(self, profile_id: str, community_id: int) -> dict | None:
+        try:
+            rows = self._db.execute(
+                "SELECT * FROM community_summaries "
+                "WHERE profile_id = ? AND community_id = ?",
+                (profile_id, int(community_id)),
+            )
+        except Exception as exc:
+            logger.debug("get_summary failed: %s", exc)
+            return None
+        return dict(rows[0]) if rows else None
+
+    # ------------------------------------------------------------------
+    # Internal
+    # ------------------------------------------------------------------
+
+    def _gather_facts(
+        self, profile_id: str, entity_to_cid: dict[str, int],
+    ) -> dict[int, list[tuple[str, str]]]:
+        """One scan → {community_id -> [(fact_id, content)]}, superseded dropped."""
+        try:
+            rows = self._db.execute(
+                "SELECT fact_id, canonical_entities_json, content "
+                "FROM atomic_facts WHERE profile_id = ?",
+                (profile_id,),
+            )
+        except Exception as exc:
+            logger.debug("community fact scan failed: %s", exc)
+            return {}
+
+        cid_facts: dict[int, list[tuple[str, str]]] = defaultdict(list)
+        for row in rows:
+            d = dict(row)
+            raw = d.get("canonical_entities_json")
+            if not raw:
+                continue
+            try:
+                ents = json.loads(raw)
+            except (ValueError, TypeError):
+                continue
+            if not isinstance(ents, list):
+                continue
+            cids = {
+                entity_to_cid[str(e).strip()]
+                for e in ents
+                if str(e).strip() in entity_to_cid
+            }
+            if not cids:
+                continue
+            fid = str(d["fact_id"])
+            content = d.get("content") or ""
+            for cid in cids:
+                cid_facts[cid].append((fid, content))
+
+        all_fids = list({fid for lst in cid_facts.values() for fid, _ in lst})
+        invalid: set[str] = set()
+        if all_fids:
+            try:
+                invalid = self._db.get_invalidated_fact_ids(all_fids, profile_id)
+            except Exception as exc:
+                logger.debug("invalidated-fact lookup failed: %s", exc)
+        if invalid:
+            cid_facts = {
+                cid: [(fid, c) for fid, c in lst if fid not in invalid]
+                for cid, lst in cid_facts.items()
+            }
+        return cid_facts
+
+    def _entity_names(
+        self, profile_id: str, communities: dict[int, list[str]],
+    ) -> dict[str, str]:
+        all_eids = list({e for ents in communities.values() for e in ents})
+        name_map: dict[str, str] = {}
+        chunk = 900
+        for start in range(0, len(all_eids), chunk):
+            batch = all_eids[start:start + chunk]
+            ph = ",".join("?" for _ in batch)
+            try:
+                rows = self._db.execute(
+                    "SELECT entity_id, canonical_name FROM canonical_entities "
+                    f"WHERE profile_id = ? AND entity_id IN ({ph})",
+                    (profile_id, *batch),
+                )
+            except Exception as exc:
+                logger.debug("entity-name lookup failed: %s", exc)
+                continue
+            for r in rows:
+                d = dict(r)
+                name_map[str(d["entity_id"])] = str(d.get("canonical_name") or "")
+        return name_map
+
+    def _keywords(self, contents: list[str]) -> str:
+        tokens: list[str] = []
+        for text in contents:
+            for word in text.lower().split():
+                w = word.strip(".,;:!?\"'()[]{}")
+                if len(w) > 2 and w not in _STOPWORDS:
+                    tokens.append(w)
+        top = [w for w, _ in Counter(tokens).most_common(self._max_keywords)]
+        return ", ".join(top)
+
+    def _summary(
+        self, contents: list[str], entity_names: list[str], keywords: str,
+    ) -> str:
+        if self._summarizer is not None:
+            try:
+                text = self._summarizer.summarize_cluster(
+                    [{"content": c} for c in contents],
+                )
+                if text and text.strip():
+                    return text.strip()[: self._summary_max_chars]
+            except Exception as exc:
+                logger.debug("community summarizer failed (fail-open): %s", exc)
+        # Mode A keyword-dense fallback.
+        names = [n for n in entity_names if n]
+        topic = ", ".join(names[:6]) if names else "related memories"
+        base = f"Topics: {topic}."
+        if keywords:
+            base += f" Key terms: {keywords}."
+        return base[: self._summary_max_chars]

package/src/superlocalmemory/core/config.py

@@ -236,6 +236,11 @@ class EncodingConfig:
     # Entropy gate
     entropy_threshold: float = 0.95
 
+    # Entity reflexion (Wave Q1) — Mode B/C self-review of extracted entities.
+    # Fail-open; one extra bounded LLM call per chunk. Mode A is unaffected.
+    enable_entity_reflexion: bool = True
+    reflexion_max_facts: int = 8
+
 
 # ---------------------------------------------------------------------------
 # Retrieval Config
@@ -297,6 +302,11 @@ class RetrievalConfig:
     recall_per_fact_max_chars: int = 2400  # ~600 tokens; head 70% + tail 30%
     recall_total_max_chars: int = 12000    # ~3K tokens; stubs beyond this
 
+    # Wave Q2b: attach a precomputed community summary as thematic context when
+    # the top results cluster in one community. Read-only lookup, gated, and
+    # fail-open — never a per-query LLM call. Kill-switch for tuning.
+    enable_community_context: bool = True
+
 
 # ---------------------------------------------------------------------------
 # Math Config
@@ -659,6 +669,14 @@ class TemporalValidatorConfig:
     # Sheaf contradiction threshold
     contradiction_threshold: float = 0.45        # Mode A threshold (768d)
 
+    # P5-INT-01: superseded facts are DEMOTED in recall, not hidden. A fact
+    # marked system_expired_at keeps its channel evidence but its per-channel
+    # score is multiplied by this factor, so current facts rank above it while
+    # nothing valid silently vanishes (Mem0-2026 non-destructive design;
+    # retrieval-time recency resolves conflicts). 0.0 restores the old hide
+    # behaviour; 1.0 disables demotion.
+    superseded_demotion_factor: float = 0.25
+
     # LLM pre-filter threshold (lower to catch more candidates)
     llm_prefilter_threshold: float = 0.30
 
@@ -677,18 +695,31 @@ class EvolutionConfig:
     """Configuration for Skill Evolution Engine (v3.4.10).
 
     OFF by default — opt in via `slm setup` (interactive) or
-    `slm config set evolution.enabled true` (CLI).
+    `slm config set evolution.enabled true` (CLI). Enabling makes
+    background LLM calls; the enable flow surfaces a cost advisory.
 
     Backend auto-detection priority:
       1. `claude` CLI available → spawn `claude --model haiku` (ECC pattern, free)
       2. Ollama running → use Ollama (free, local)
       3. API key set → use Anthropic/OpenAI API (paid)
       4. Nothing → dashboard-only (show candidates, manual evolution)
+
+    Model selection (v3.7.9): each pipeline step is configurable and
+    defaults to the lowest-cost capable model for the active backend
+    (Claude→haiku, Ollama→local). Leave a field empty ("") for "auto".
+    The blind verifier is kept on a *different* model from the generator
+    so it can't grade its own homework; see
+    ``evolution.model_selection.resolve_evolution_models``.
     """
 
     enabled: bool = False                        # OFF by default, opt-in
     backend: str = "auto"                        # auto, claude, ollama, anthropic, openai
     max_evolutions_per_cycle: int = 3            # Budget cap per consolidation
+    # Empty string == "auto" (resolve cheapest capable model at runtime).
+    # Accepts short aliases ("haiku"/"sonnet") or allow-listed model ids.
+    mutation_model: str = ""                     # generator (quality-sensitive)
+    verify_model: str = ""                       # blind verifier (must differ from generator)
+    confirm_model: str = ""                      # cheap yes/no gate
 
 
 @dataclass(frozen=True)
@@ -743,6 +774,152 @@ class AutoInvokeConfig:
     relevance_threshold: float = 0.3           # Legacy compat with AutoRecall
 
 
+# ---------------------------------------------------------------------------
+# Deployment Config (v3.8.0)
+# ---------------------------------------------------------------------------
+
+_VALID_DEPLOYMENT_MODES = ("personal", "enterprise")
+
+
+@dataclass(frozen=True)
+class DeploymentConfig:
+    """Deployment mode configuration.
+
+    Personal (default, safe defaults):
+        Single-user install. No login required, no PII redaction, no
+        retention scheduler. Behaviour is identical to pre-3.8.0 installs.
+
+    Enterprise:
+        Multi-user / team / company install. Login required, PII redacted,
+        retention scheduler active, audit enabled.
+
+    SAFE DEFAULTS = Personal — an existing config.toml with no [deployment]
+    section continues to behave exactly as before.  Adding a [deployment]
+    section is opt-in; removing it reverts to Personal automatically.
+    """
+
+    mode: str = "personal"          # "personal" | "enterprise"
+    require_login: bool = False
+    pii_redaction: bool = False
+    retention_enabled: bool = False
+    audit: bool = True
+
+    def __post_init__(self) -> None:
+        if self.mode not in _VALID_DEPLOYMENT_MODES:
+            raise ValueError(
+                f"DeploymentConfig.mode must be one of {_VALID_DEPLOYMENT_MODES!r}, "
+                f"got {self.mode!r}"
+            )
+
+    @property
+    def is_personal(self) -> bool:
+        """True when operating in personal (single-user) mode."""
+        return self.mode == "personal"
+
+    @property
+    def is_enterprise(self) -> bool:
+        """True when operating in enterprise (team/company) mode."""
+        return self.mode == "enterprise"
+
+    def as_dict(self) -> dict:
+        return {
+            "mode": self.mode,
+            "require_login": self.require_login,
+            "pii_redaction": self.pii_redaction,
+            "retention_enabled": self.retention_enabled,
+            "audit": self.audit,
+        }
+
+
+#: Canonical Personal preset — all permissive defaults.
+DEPLOYMENT_PERSONAL = DeploymentConfig(
+    mode="personal",
+    require_login=False,
+    pii_redaction=False,
+    retention_enabled=False,
+    audit=True,
+)
+
+#: Canonical Enterprise preset — all enforcement defaults.
+DEPLOYMENT_ENTERPRISE = DeploymentConfig(
+    mode="enterprise",
+    require_login=True,
+    pii_redaction=True,
+    retention_enabled=True,
+    audit=True,
+)
+
+
+def load_deployment_config(
+    config_toml_path: Path | None = None,
+) -> DeploymentConfig:
+    """Parse [deployment] from config.toml; return Personal defaults if absent.
+
+    config.toml is the installer-written performance config (separate from the
+    daemon's config.json managed by SLMConfig). This function reads ONLY the
+    [deployment] section. All other sections are ignored.
+
+    Args:
+        config_toml_path: Explicit path to config.toml. When None, resolved to
+            ``~/.superlocalmemory/config.toml`` via the canonical data root.
+
+    Returns:
+        DeploymentConfig — Personal preset when the file is absent, the section
+        is missing, or any parse error occurs (fail-open, non-destructive).
+    """
+    if config_toml_path is None:
+        try:
+            config_toml_path = _runtime_base_dir() / "config.toml"
+        except Exception:
+            return DEPLOYMENT_PERSONAL
+
+    if not config_toml_path.exists():
+        return DEPLOYMENT_PERSONAL
+
+    try:
+        import tomllib as _tomllib
+        raw = config_toml_path.read_text(encoding="utf-8")
+        data = _tomllib.loads(raw)
+    except Exception as exc:
+        logger.warning(
+            "load_deployment_config: failed to parse %s: %s", config_toml_path, exc
+        )
+        return DEPLOYMENT_PERSONAL
+
+    dep = data.get("deployment", {})
+    if not dep:
+        # No [deployment] section — personal defaults, no behaviour change.
+        return DEPLOYMENT_PERSONAL
+
+    raw_mode = str(dep.get("mode", "personal")).lower()
+    if raw_mode not in _VALID_DEPLOYMENT_MODES:
+        logger.warning(
+            "load_deployment_config: unknown mode %r in %s — defaulting to personal",
+            raw_mode, config_toml_path,
+        )
+        raw_mode = "personal"
+
+    # Use the preset for the mode as the base so omitted keys inherit
+    # sensible values (enterprise → require_login=True etc.).
+    base = DEPLOYMENT_ENTERPRISE if raw_mode == "enterprise" else DEPLOYMENT_PERSONAL
+
+    try:
+        return DeploymentConfig(
+            mode=raw_mode,
+            require_login=bool(dep.get("require_login", base.require_login)),
+            pii_redaction=bool(dep.get("pii_redaction", base.pii_redaction)),
+            retention_enabled=bool(dep.get("retention_enabled", base.retention_enabled)),
+            audit=bool(dep.get("audit", base.audit)),
+        )
+    except (ValueError, TypeError) as exc:
+        logger.warning(
+            "load_deployment_config: invalid values in [deployment] in %s: %s — "
+            "falling back to personal",
+            config_toml_path, exc,
+        )
+        return DEPLOYMENT_PERSONAL
+
+
 # ---------------------------------------------------------------------------
 # Health Config (v3.6.9 BUG-A)
 # ---------------------------------------------------------------------------
@@ -849,6 +1026,35 @@ class SLMConfig:
         path = config_path or (_runtime_base / "config.json")
         if not path.exists():
             return cls.for_mode(Mode.A, base_dir=_runtime_base)
+
+        # DASH-V4 (3.7.9): the ``current_mode`` file is the single source of
+        # truth for the ACTIVE mode. If config.json drifted from it (a writer
+        # bypassed switch_mode and reset the active config), load the
+        # authoritative per-mode file so daemon/CLI/dashboard all run the user's
+        # chosen mode — with its settings — instead of a stale one. Scoped to
+        # config.json so explicit per-mode loads (switch_mode) are untouched.
+        # Fail-open: any error falls through to the normal load below.
+        if path.name == "config.json":
+            try:
+                import json as _json
+                _disk_mode = str(
+                    _json.loads(path.read_text()).get("mode", "")
+                ).lower()
+                _active_mode = cls.read_current_mode(path.parent)
+                if _disk_mode and _active_mode and _disk_mode != _active_mode:
+                    _mode_path = cls._mode_config_path(
+                        path.parent, Mode(_active_mode)
+                    )
+                    if _mode_path.exists():
+                        logger.warning(
+                            "config.json mode=%s disagrees with current_mode=%s;"
+                            " loading authoritative %s",
+                            _disk_mode, _active_mode, _mode_path.name,
+                        )
+                        return cls.load(_mode_path)
+            except Exception:
+                pass  # fail-open — fall through to the normal load below
+
         import json
         try:
             data = json.loads(path.read_text())
@@ -1070,6 +1276,9 @@ class SLMConfig:
             "enabled": self.evolution.enabled,
             "backend": self.evolution.backend,
             "max_evolutions_per_cycle": self.evolution.max_evolutions_per_cycle,
+            "mutation_model": self.evolution.mutation_model,
+            "verify_model": self.evolution.verify_model,
+            "confirm_model": self.evolution.confirm_model,
         }
 
         # V3.4.65: Persist injection config
@@ -1264,6 +1473,7 @@ class SLMConfig:
             _c_emb = EmbeddingConfig(
                 model_name=embedding_model_name or "text-embedding-3-large",
                 dimension=embedding_dimension or 3072,
+                provider=_c_emb_provider,
                 api_endpoint=embedding_endpoint,
                 api_key=embedding_key,
                 deployment_name=embedding_deployment,
@@ -1275,8 +1485,11 @@ class SLMConfig:
             # 768-dim Ollama embedder, which made ingestion fail at vector
             # materialization. Cloud embeddings remain an explicit opt-in.
             _c_emb = EmbeddingConfig(
-                model_name="nomic-ai/nomic-embed-text-v1.5",
-                dimension=768,
+                # Honour an on-disk embedding model when one was configured (the
+                # load() path passes it through); default to the local nomic
+                # model so Mode C never silently requires a paid cloud embedder.
+                model_name=embedding_model_name or "nomic-ai/nomic-embed-text-v1.5",
+                dimension=embedding_dimension or 768,
             )
         return cls(
             mode=mode,