superlocalmemory 4.0.8 → 4.0.10

package/src/superlocalmemory/core/install_detector.py

@@ -0,0 +1,131 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under AGPL-3.0-or-later - see LICENSE file
+
+"""Detect all SuperLocalMemory installations present on this machine.
+
+Supports three install types: pipx, venv (~/.slm-venv), and npm global.
+All detection is read-only and fast (< 200 ms). No writes are ever performed.
+"""
+
+from __future__ import annotations
+
+import glob
+import json
+import subprocess
+import sys
+from pathlib import Path
+from typing import Optional
+
+# Roots for the two Python install types. Patched in tests.
+_VENV_ROOT: Path = Path.home() / ".slm-venv"
+_PIPX_ROOT: Path = Path.home() / ".local" / "pipx" / "venvs" / "superlocalmemory"
+
+
+def _npm_global_root() -> Optional[Path]:
+    """Return the npm global node_modules root, or None on any failure."""
+    try:
+        result = subprocess.run(
+            ["npm", "root", "-g"],
+            capture_output=True,
+            text=True,
+            timeout=5,
+        )
+        if result.returncode == 0:
+            root = result.stdout.strip()
+            if root:
+                return Path(root)
+    except Exception:  # npm absent, timeout, etc.
+        pass
+    return None
+
+
+def _read_python_version(base: Path) -> Optional[str]:
+    """Read __version__ from the first matching site-packages layout under base.
+
+    Both layouts are searched. POSIX virtualenvs use
+    ``lib/python3.13/site-packages``; Windows uses ``Lib/site-packages`` with no
+    version component and a capitalised directory. Searching only the POSIX
+    shape made detection silently return None on Windows — which is precisely
+    where multi-install divergence between pip and npm is most likely, and where
+    the version-mismatch error would then name no installations at all.
+    """
+    patterns = [
+        str(base / "lib" / "python*" / "site-packages" / "superlocalmemory" / "__init__.py"),
+        str(base / "Lib" / "site-packages" / "superlocalmemory" / "__init__.py"),
+        str(base / "lib" / "site-packages" / "superlocalmemory" / "__init__.py"),
+    ]
+    matches: list[str] = []
+    for pattern in patterns:
+        matches.extend(glob.glob(pattern))
+    for init_path in sorted(set(matches)):
+        try:
+            text = Path(init_path).read_text(encoding="utf-8", errors="replace")
+            for line in text.splitlines():
+                line = line.strip()
+                if line.startswith("__version__"):
+                    # __version__ = "4.1.0"  or  __version__ = '4.1.0'
+                    parts = line.split("=", 1)
+                    if len(parts) == 2:
+                        return parts[1].strip().strip("\"'")
+        except OSError:
+            continue
+    return None
+
+
+def _read_npm_version(npm_root: Path) -> Optional[str]:
+    """Read version from npm global package.json."""
+    pkg_json = npm_root / "superlocalmemory" / "package.json"
+    try:
+        data = json.loads(pkg_json.read_text(encoding="utf-8"))
+        return str(data.get("version", "")).strip() or None
+    except (OSError, json.JSONDecodeError):
+        return None
+
+
+def _detect_all_installs() -> list[dict]:
+    """Return all SuperLocalMemory installs detected on this machine.
+
+    Each entry is a dict with keys:
+      - ``path``    (str) — directory of the install
+      - ``version`` (str) — version string read from package metadata
+      - ``type``    (str) — one of "pipx", "venv", "npm"
+
+    Detection is read-only and best-effort. A missing or unreadable install
+    produces no entry rather than an error. Subprocess calls are bounded to
+    5 seconds total.
+    """
+    results: list[dict] = []
+
+    # --- pipx ---
+    pipx_version = _read_python_version(_PIPX_ROOT)
+    if pipx_version is not None:
+        results.append({
+            "path": str(_PIPX_ROOT) + "/",
+            "version": pipx_version,
+            "type": "pipx",
+        })
+
+    # --- ~/.slm-venv ---
+    venv_version = _read_python_version(_VENV_ROOT)
+    if venv_version is not None:
+        results.append({
+            "path": str(_VENV_ROOT) + "/",
+            "version": venv_version,
+            "type": "venv",
+        })
+
+    # --- npm global ---
+    npm_root = _npm_global_root()
+    if npm_root is not None:
+        npm_version = _read_npm_version(npm_root)
+        if npm_version is not None:
+            results.append({
+                "path": str(npm_root / "superlocalmemory") + "/",
+                "version": npm_version,
+                "type": "npm",
+            })
+
+    return results
+
+
+__all__ = ["_detect_all_installs"]

package/src/superlocalmemory/core/maintenance.py

@@ -31,6 +31,16 @@ if TYPE_CHECKING:
 
 logger = logging.getLogger(__name__)
 
+
+class _ConsolidationDisabled(Exception):
+    """Internal signal: consolidation is switched off, so skip its block.
+
+    A private exception rather than restructuring the surrounding try/except:
+    the block's job is to keep one optional maintenance step from taking the
+    whole pass down with it, and that guarantee should not be weakened to
+    express "deliberately skipped". Caught immediately below, never propagated.
+    """
+
 # Backfill constants
 _BACKFILL_BURN_IN_STEPS = 50
 _LANGEVIN_DIM = 8
@@ -638,9 +648,12 @@ def run_maintenance(
         logger.warning("Entity summary consolidation failed: %s", exc)
 
     # 4. Fact consolidation (v3.8.4 concurrency-safe path via DatabaseManager).
-    # Merges clusters of warm/cold atomic facts about the same entity into a
-    # single consolidated fact, archives the originals (NEVER deletes them),
-    # and records provenance in fact_consolidations.
+    # Groups warm/cold atomic facts that share an entity and writes ONE
+    # DISPLAY summary per cluster into consolidated_summaries, with provenance
+    # in fact_consolidations. It does not write to atomic_facts and does not
+    # archive the source facts — until 4.0.10 it did both, which put 1,195
+    # model-written rows into the retrieval corpus and left 528 genuine
+    # memories archived out of normal recall.
     #
     # Uses the DatabaseManager path so LLM calls happen OUTSIDE the write lock:
     #   - Discover clusters in a short memory_read() (no write lock held).
@@ -652,20 +665,45 @@ def run_maintenance(
     try:
         from superlocalmemory.core.fact_consolidator import consolidate_facts
 
+        # The documented off-switch has to actually switch something off.
+        # ConsolidationConfig.enabled has existed since Phase 5 and this call
+        # site never read it, so a user who ran `slm config` to turn
+        # consolidation off got consolidation anyway — for four months, on
+        # every maintenance pass. -2 is a third distinguishable value, kept
+        # apart from 0 (nothing to merge) and -1 (the step failed), so a
+        # deliberately disabled step is never mistaken for either.
+        _consolidation = getattr(config, "consolidation", None)
+        if _consolidation is not None and not getattr(_consolidation, "enabled", True):
+            counts["facts_consolidated"] = -2
+            logger.debug("Fact consolidation disabled by configuration")
+            raise _ConsolidationDisabled
+
         fc_stats = consolidate_facts(
             db,
             profile_id=profile_id,
-            max_clusters=getattr(config, "max_consolidation_clusters", 20),
+            # Read from ConsolidationConfig, with the old SLMConfig-level name
+            # as the fallback. `getattr(config, "max_consolidation_clusters")`
+            # alone never resolved — SLMConfig has no such attribute — so the
+            # default was the only value this had ever used.
+            max_clusters=int(
+                getattr(_consolidation, "max_consolidation_clusters", None)
+                or getattr(config, "max_consolidation_clusters", None)
+                or 20
+            ),
             dry_run=False,
             config=config,
         )
         counts["facts_consolidated"] = fc_stats.get("consolidated", 0)
         if fc_stats.get("consolidated", 0) > 0:
             logger.info(
-                "Fact consolidation: %d clusters merged, %d facts archived",
+                "Fact consolidation: %d display summaries over %d facts "
+                "(%d clusters refused)",
                 fc_stats.get("consolidated", 0),
-                fc_stats.get("facts_archived", 0),
+                fc_stats.get("facts_summarized", 0),
+                fc_stats.get("rejected", 0),
             )
+    except _ConsolidationDisabled:
+        pass
     except Exception as exc:
         # WARNING, not debug, and a distinguishable count. Leaving this at debug
         # with facts_consolidated=0 made a failing consolidation report exactly

package/src/superlocalmemory/core/memory_health.py

@@ -0,0 +1,266 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under AGPL-3.0-or-later - see LICENSE file
+
+"""Tell the owner, in their own words, whether their memory works.
+
+Until now the only way to learn that 43.7% of a store could not be found by
+asking a question was to write the SQL yourself. One machine sat in exactly
+that state for months while every status line it showed said the system was
+healthy, because nothing measured reachability and nothing reported it.
+
+So this module answers four questions a non-engineer can act on:
+
+  * How many memories do I have?
+  * How many can actually be found by asking a question?
+  * How many were withheld because a model wrote them, not me?
+  * Is anything still being repaired?
+
+Read-only, and every query is bounded. Fail-soft by construction: a missing
+table or column yields ``None`` for that line rather than an exception, because
+a health report that crashes on an old store is worse than one that says "not
+known yet".
+
+Consumed by ``slm doctor``, ``GET /api/v3/memory-health``, and the dashboard.
+One implementation so the three cannot disagree with each other.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+import sqlite3
+from dataclasses import dataclass, field
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+__all__ = ["MemoryHealth", "measure", "describe"]
+
+
+@dataclass(frozen=True)
+class MemoryHealth:
+    """A store's answer-ability, counted rather than assumed."""
+
+    #: Memories that recall is allowed to return.
+    live_facts: int = 0
+    #: Of those, how many have a vector projection, i.e. can be found by
+    #: meaning rather than only by matching words.
+    findable_by_meaning: int = 0
+    #: Memories with no vector at all. These are reachable by keyword only.
+    missing_vector: int = 0
+    #: Machine-written summaries withheld from recall and kept for display.
+    withheld_summaries: int = 0
+    #: Summaries preserved in the display table.
+    display_summaries: int = 0
+    #: Memories hidden by the retention system, excluding the withheld ones.
+    hidden_by_forgetting: int = 0
+    #: Rows whose retention zone contradicts their retention score, i.e. hidden
+    #: while the maths says to keep them. Should be zero after repair.
+    inconsistently_hidden: int = 0
+    #: Present only when a table or column was absent.
+    unavailable: tuple[str, ...] = field(default_factory=tuple)
+
+    @property
+    def reachability(self) -> float:
+        """Share of live memories findable by meaning, 0.0-1.0."""
+        if self.live_facts <= 0:
+            return 1.0
+        return self.findable_by_meaning / self.live_facts
+
+    @property
+    def healthy(self) -> bool:
+        """Whether anything here warrants telling the owner about."""
+        return (
+            self.reachability >= 0.99
+            and self.missing_vector == 0
+            and self.inconsistently_hidden == 0
+        )
+
+
+def measure(db_path: str | Path) -> MemoryHealth:
+    """Count the store's answer-ability. Read-only; never raises."""
+    unavailable: list[str] = []
+    try:
+        conn = sqlite3.connect(f"file:{Path(db_path)}?mode=ro", uri=True)
+    except sqlite3.Error as exc:
+        logger.debug("memory health: cannot open %s: %s", db_path, exc)
+        return MemoryHealth(unavailable=("database",))
+
+    try:
+        if not _table_exists(conn, "atomic_facts"):
+            return MemoryHealth(unavailable=("atomic_facts",))
+
+        # Quarantine came in 4.0.10. On an older store every fact is "live",
+        # which is the honest reading of a store that has no withheld rows.
+        has_q = _has_column(conn, "atomic_facts", "quarantined")
+        if not has_q:
+            unavailable.append("quarantined")
+        live_clause = "COALESCE(quarantined, 0) = 0" if has_q else "1=1"
+
+        live = _count(conn, f"SELECT COUNT(*) FROM atomic_facts WHERE {live_clause}")
+        withheld = (
+            _count(conn, "SELECT COUNT(*) FROM atomic_facts WHERE quarantined = 1")
+            if has_q else 0
+        )
+        missing_vec = _count(
+            conn,
+            f"SELECT COUNT(*) FROM atomic_facts "
+            f"WHERE embedding IS NULL AND {live_clause}",
+        )
+
+        if _table_exists(conn, "embedding_metadata"):
+            findable = _count(
+                conn,
+                "SELECT COUNT(*) FROM embedding_metadata em "
+                "JOIN atomic_facts af ON af.fact_id = em.fact_id "
+                f"WHERE {_prefixed(live_clause, 'af')}",
+            )
+        else:
+            unavailable.append("embedding_metadata")
+            findable = 0
+
+        display = (
+            _count(conn, "SELECT COUNT(*) FROM consolidated_summaries")
+            if _table_exists(conn, "consolidated_summaries") else 0
+        )
+        if not _table_exists(conn, "consolidated_summaries"):
+            unavailable.append("consolidated_summaries")
+
+        hidden = inconsistent = 0
+        if _table_exists(conn, "fact_retention"):
+            hidden = _count(
+                conn,
+                "SELECT COUNT(*) FROM fact_retention r "
+                "JOIN atomic_facts af ON af.fact_id = r.fact_id "
+                "WHERE r.lifecycle_zone IN ('archive', 'forgotten') "
+                f"  AND {_prefixed(live_clause, 'af')}",
+            )
+            # The contradiction M043 repairs: hidden, yet scored to keep.
+            inconsistent = _count(
+                conn,
+                "SELECT COUNT(*) FROM fact_retention r "
+                "JOIN atomic_facts af ON af.fact_id = r.fact_id "
+                "WHERE r.lifecycle_zone IN ('archive', 'forgotten') "
+                "  AND r.retention_score > 0.8 "
+                f"  AND {_prefixed(live_clause, 'af')}",
+            )
+        else:
+            unavailable.append("fact_retention")
+
+        return MemoryHealth(
+            live_facts=live,
+            findable_by_meaning=findable,
+            missing_vector=missing_vec,
+            withheld_summaries=withheld,
+            display_summaries=display,
+            hidden_by_forgetting=hidden,
+            inconsistently_hidden=inconsistent,
+            unavailable=tuple(unavailable),
+        )
+    except sqlite3.Error as exc:
+        logger.debug("memory health measurement failed: %s", exc)
+        return MemoryHealth(unavailable=(*unavailable, "query_failed"))
+    finally:
+        conn.close()
+
+
+def describe(health: MemoryHealth) -> list[str]:
+    """Plain-language lines for a reader who does not write SQL.
+
+    No percentages without the counts behind them, and no jargon: "findable by
+    asking a question" rather than "vector coverage", because the person who
+    needs this line is the one who would not know what a vector is.
+    """
+    lines: list[str] = []
+    if "atomic_facts" in health.unavailable or "database" in health.unavailable:
+        return ["Memory store not readable yet."]
+
+    lines.append(f"You have {health.live_facts:,} memories.")
+
+    if "embedding_metadata" in health.unavailable:
+        lines.append(
+            "Whether they can be found by asking a question is not known yet — "
+            "the search index has not been built."
+        )
+    elif health.live_facts:
+        pct = 100.0 * health.reachability
+        if health.findable_by_meaning >= health.live_facts:
+            # "All" only when the counts actually agree. The threshold used to
+            # be reachability >= 0.99, which printed "All of them can be found
+            # by asking a question (5,199 indexed)" on a store of 5,205 — a
+            # claim of all, contradicted by the number beside it. This module
+            # exists to be believed; it cannot round in its own favour.
+            lines.append(
+                f"All {health.live_facts:,} of them can be found by asking a "
+                f"question."
+            )
+        elif health.reachability >= 0.99:
+            gap = health.live_facts - health.findable_by_meaning
+            lines.append(
+                f"{health.findable_by_meaning:,} of them can be found by asking "
+                f"a question. The other {gap:,} can only be found by matching "
+                f"words. That is a small enough share to be normal — a memory "
+                f"written moments ago, or one the model could not read."
+            )
+        else:
+            gap = health.live_facts - health.findable_by_meaning
+            lines.append(
+                f"{health.findable_by_meaning:,} of them ({pct:.0f}%) can be "
+                f"found by asking a question. The other {gap:,} can only be "
+                f"found by matching words, so a question phrased differently "
+                f"will miss them. This repairs itself as the service runs; if "
+                f"it does not, the embedding model is unavailable."
+            )
+
+    if health.withheld_summaries:
+        lines.append(
+            f"{health.withheld_summaries:,} machine-written summaries are kept "
+            f"out of your answers and shown on the dashboard instead. They were "
+            f"written by the summarizer, not by you, and they used to be "
+            f"returned as if they were your own notes."
+        )
+
+    if health.inconsistently_hidden:
+        lines.append(
+            f"{health.inconsistently_hidden:,} memories are hidden even though "
+            f"they are marked worth keeping. This is a fault and it is repaired "
+            f"automatically the next time the service starts."
+        )
+
+    if health.hidden_by_forgetting:
+        lines.append(
+            f"{health.hidden_by_forgetting:,} older memories are set aside by "
+            f"the forgetting curve. They are not deleted and a deep search "
+            f"still reaches them."
+        )
+
+    return lines
+
+
+def _table_exists(conn: sqlite3.Connection, table: str) -> bool:
+    return conn.execute(
+        "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (table,),
+    ).fetchone() is not None
+
+
+def _has_column(conn: sqlite3.Connection, table: str, column: str) -> bool:
+    return any(
+        row[1] == column for row in conn.execute(f"PRAGMA table_info({table})")
+    )
+
+
+def _prefixed(clause: str, prefix: str) -> str:
+    """Qualify a bare column reference for use in a joined query.
+
+    Word-bounded, so a future column named ``quarantined_at`` is not silently
+    rewritten to ``af.quarantined_at`` by a substring match. No such column
+    exists today; the point is that the failure would be a wrong count rather
+    than an error, and a wrong count in a health report is the one thing this
+    module must not produce.
+    """
+    return re.sub(r"\bquarantined\b", f"{prefix}.quarantined", clause)
+
+
+def _count(conn: sqlite3.Connection, sql: str) -> int:
+    row = conn.execute(sql).fetchone()
+    return int(row[0]) if row else 0

package/src/superlocalmemory/core/operation_policy_registry.py

@@ -281,7 +281,7 @@ class OperationPolicyRegistry:
                 allowed_transports=_ALL_TRANSPORTS,
                 audit_level="full",
             ),
-            # Operational recovery & admin remediation (Wave-3 resilience slice)
+            # Operational recovery & admin remediation (resilience slice)
             # OPS_INSPECT: read-only listing of failed/stuck/degraded ops.
             # Allowed over all transports so dashboard, MCP, and CLI all work.
             OperationKind.OPS_INSPECT: OperationPolicy(

package/src/superlocalmemory/core/operation_request.py

@@ -51,7 +51,7 @@ class OperationKind(str, Enum):
     SCHEMA_MIGRATE = "schema_migrate"
     VECTOR_MIGRATE = "vector_migrate"
     EVOLVE_SKILL = "evolve_skill"
-    # Operational recovery & admin remediation (V4 Wave-3 resilience slice)
+    # Operational recovery & admin remediation (V4 resilience slice)
     OPS_INSPECT = "ops_inspect"    # List failed/stuck/degraded operations (OWNER/ADMIN)
     OPS_RESOLVE = "ops_resolve"    # Retry/force-reconcile/cancel an operation (OWNER/ADMIN)
 

package/src/superlocalmemory/core/ops_remediation.py

@@ -1,7 +1,7 @@
 # Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
 # Licensed under AGPL-3.0-or-later - see LICENSE file
 
-"""Operational Recovery & Admin Remediation helpers (Wave-3 resilience slice).
+"""Operational Recovery & Admin Remediation helpers (resilience slice).
 
 Provides two primary functions used by HTTP endpoints, MCP tools, and CLI:
 
@@ -22,7 +22,7 @@ Design constraints (NON-NEGOTIABLE):
     - Additive & backward-compatible — healthy-path unaffected
     - Immutable return dicts; explicit error handling; no silent swallowing
 
-Part of SuperLocalMemory V4 | Wave-3: Operational Recovery
+Part of SuperLocalMemory V4 | Operational Recovery
 """
 
 from __future__ import annotations

package/src/superlocalmemory/core/progressive_abstraction.py

@@ -16,7 +16,7 @@ The persona is one bounded roll-up per profile that consumes the top community
 summaries. It is recall-GATED (never auto-injected into the hot recall path —
 avoids the V3.4.40 summary-pollution regression) and SIZE-bounded. Drill-down
 (``get_sources``) walks the hierarchy back down to the source atoms, matching
-the market bar for summary->source provenance (Zep-style).
+the design goal for summary->source provenance (Zep-style attribution).
 
 Runs in the background consolidation lane after community summaries.
 Fail-open throughout; recompute replaces a profile's row.

package/src/superlocalmemory/core/security_primitives.py

@@ -4,8 +4,6 @@
 
 """Shared security primitives for SLM v3.4.22.
 
-LLD reference: `.backup/active-brain/lld/LLD-07-schema-migrations-and-security-primitives.md`
-Section: 6.1 through 6.10.
 
 Every file write, subprocess spawn, and secret-bearing string across SLM
 daemon, adapters, hooks, and binary installer routes through this module.
@@ -155,10 +153,9 @@ def safe_resolve(base: Path, rel: str | Path) -> Path:
 # enforces the LLD-00 regex AND the base-containment check. Callers in
 # LLD-09 (session state files) and LLD-11 (evolution.lock) MUST use this.
 #
-# Naming deviation from IMPLEMENTATION-MANIFEST P0.2: the manifest reused
-# the name ``safe_resolve`` but the existing path-style helper is used in
-# 9+ call sites. A separate name avoids breakage. See
-# ``.backup/active-brain/MANIFEST-DEVIATION.md`` P0.2 entry.
+# Naming deviation note: the name ``safe_resolve`` was already used by an
+# existing path-style helper that has 9+ call sites. A separate name
+# (``safe_resolve_identifier``) avoids silent breakage at those call sites.
 
 _SAFE_ID_RE = re.compile(r"^[a-zA-Z0-9_-]{1,128}$")