superlocalmemory 4.0.9 → 4.0.10

package/src/superlocalmemory/server/routes/abstraction.py

@@ -9,6 +9,8 @@ down to source atoms:
 
   GET /api/v3/abstraction/persona      — the per-profile persona roll-up
   GET /api/v3/abstraction/communities  — community summaries (Q2)
+  GET /api/v3/abstraction/consolidated — display-only cluster summaries
+  GET /api/v3/abstraction/health       — can my memories be found? (4.0.10)
   GET /api/v3/abstraction/sources      — drill-down (node -> source atoms)
 
 Read-only, profile-scoped (Rule 01), direct sqlite3 (Rule 06). All handlers
@@ -30,6 +32,31 @@ logger = logging.getLogger(__name__)
 
 router = APIRouter(prefix="/api/v3/abstraction", tags=["abstraction"])
 
+#: How many summary rows /consolidated will read before ranking them by
+#: quality. Bounded because this runs on a request thread: a store with tens of
+#: thousands of summaries must not turn one card into a full-table scan.
+_SCAN_CEILING = 400
+
+#: Characters of normalised opening text that make two summaries "the same
+#: summary" for display. Long enough that two genuinely different subjects
+#: diverge within it, short enough to catch the same sentence with a different
+#: tail — which is the shape the summarizer actually produces.
+_OPENING_KEY_CHARS = 90
+
+
+def _opening_key(content: object) -> str:
+    """Normalised opening of a summary, for near-duplicate collapsing.
+
+    Case-folded with runs of whitespace flattened, so two summaries differing
+    only in line wrapping or capitalisation collapse together. Returns "" for
+    anything too short to judge, which is then never collapsed — better to show
+    a duplicate than to hide a distinct summary on a weak signal.
+    """
+    text = " ".join(str(content or "").split()).casefold()
+    if len(text) < 40:
+        return ""
+    return text[:_OPENING_KEY_CHARS]
+
 
 class _ReadDB:
     """Adapt a raw sqlite3 connection to the .execute(...) -> list contract
@@ -111,3 +138,177 @@ def get_sources(
         return JSONResponse({"profile": pid, "sources": empty})
     finally:
         conn.close()
+
+
+@router.get("/consolidated")
+def get_consolidated(
+    profile: str = Query(""),
+    limit: int = Query(50, ge=1, le=200),
+    include_unusable: bool = Query(False),
+) -> JSONResponse:
+    """Cluster summaries, read from the DISPLAY table and nowhere else.
+
+    ``consolidated_summaries`` is the only source. Reading ``atomic_facts``
+    here would put the boundary back where it was: these summaries were in the
+    retrieval corpus until 4.0.10 and the whole point of moving them is that
+    exactly one surface shows them, and it is this one.
+
+    ``summaries`` holds only rows worth reading. The rest are REPORTED, not
+    returned: ``unusable`` and ``near_duplicates`` are counts over the scanned
+    window. A reader is better served by "62 of these came back empty" than by a
+    page that silently shows a handful and looks complete — and hiding the fact
+    that they came back empty would hide the problem this endpoint exists to make
+    visible. ``include_unusable=true`` returns them for inspection.
+
+    Two orderings, both measured rather than chosen:
+
+    * Ranking by ``source_count`` alone put the junk on top, because the
+      summaries merging the largest clusters are exactly the ones the model had
+      least in common to work with. On the author's store **0 of the top 24 by
+      source_count were usable**, so a card asking for 24 rendered empty against
+      a store holding a thousand summaries.
+    * Rows covering real memories rank above rows covering none. 353 of these
+      are summaries of summaries; their honest ``source_count`` is 0, and a
+      digest of the summarizer's own output is worth less to a reader than a
+      digest of their own words.
+
+    Quality is a Python predicate rather than a SQL expression, which is why the
+    window is read to at most ``_SCAN_CEILING`` rows, classified, and then
+    ordered.
+    """
+    pid = profile or get_active_profile()
+    conn = _conn()
+    if conn is None:
+        return JSONResponse({
+            "profile": pid, "summaries": [], "unusable": 0, "scanned": 0,
+        })
+    try:
+        from superlocalmemory.summaries.base import clean_llm_summary
+        from superlocalmemory.summaries.non_answer import (
+            MIN_USEFUL_CHARS,
+            is_non_answer,
+        )
+
+        scan = min(_SCAN_CEILING, max(int(limit) * 8, int(limit)))
+        rows = conn.execute(
+            "SELECT summary_id, entity_name, content, source_count, "
+            "       generated_by, source_earliest, source_latest, created_at "
+            "  FROM consolidated_summaries "
+            " WHERE profile_id = ? "
+            " ORDER BY source_count DESC, created_at DESC, summary_id ASC "
+            " LIMIT ?",
+            (pid, scan),
+        ).fetchall()
+
+        classified: list[dict[str, Any]] = []
+        unusable = 0
+        for row in rows:
+            item = dict(row)
+            # CLEAN, THEN JUDGE — the same order the write path uses, and for
+            # the same reason. Rows migrated from the old corpus were never
+            # cleaned, so their scaffolding is still attached: judging first let
+            # "Here is a concise summary paragraph incorporating all 10 facts..."
+            # through as usable and put it at the top of the card, because it
+            # does contain a summary and the non-answer rules are about refusals,
+            # not preambles. Cleaning is also what the reader should see: the
+            # scaffolding is addressed to a conversation they cannot read.
+            item["content"] = clean_llm_summary(str(item.get("content") or ""))
+            rejected, why = is_non_answer(
+                item["content"], min_chars=MIN_USEFUL_CHARS,
+            )
+            item["quality"] = why if rejected else "ok"
+            if rejected:
+                unusable += 1
+            classified.append(item)
+
+        # Usable first, then rows that cover real memories, then the SQL
+        # ordering within each group. Stable sort, so two runs of one request
+        # return the same rows in the same order — a summary card that
+        # reshuffles itself on refresh reads as a bug even when every row is
+        # correct.
+        classified.sort(key=lambda item: (
+            0 if item["quality"] == "ok" else 1,
+            0 if (item.get("source_count") or 0) > 0 else 1,
+        ))
+
+        # Collapse near-duplicates.
+        #
+        # The table's UNIQUE constraint is on exact content, so summaries that
+        # differ by a clause survive as separate rows. On the author's store the
+        # first 24 usable rows all opened "The Pro and SuperLocalMemory (SLM)
+        # projects have made significant progress in..." — twenty-four cards
+        # saying one thing, which reads as a broken page rather than as a view
+        # of a memory.
+        #
+        # Collapsed on a normalised opening, keeping the row that merged the
+        # most memories (the ordering above already put it first). The count is
+        # reported, not swallowed: that these summaries repeat each other is a
+        # real property of the store and worth a reader knowing.
+        deduped: list[dict[str, Any]] = []
+        seen_openings: set[str] = set()
+        collapsed = 0
+        for item in classified:
+            key = _opening_key(item.get("content"))
+            if key and key in seen_openings:
+                collapsed += 1
+                continue
+            if key:
+                seen_openings.add(key)
+            deduped.append(item)
+
+        # Only rows worth reading occupy the window.
+        #
+        # A first draft returned everything, usable first, and truncated at
+        # `limit`. Because the usable rows on this store collapse to a handful
+        # of distinct openings, the tail of a limit-10 request filled with
+        # refusals — and a card asking for 10 got 2 it could render and 8 it
+        # threw away. The counts carry what the reader needs to know about the
+        # rest; the rows themselves add nothing to a card.
+        shown = (
+            deduped[:int(limit)] if include_unusable
+            else [i for i in deduped if i["quality"] == "ok"][:int(limit)]
+        )
+        return JSONResponse({
+            "profile": pid,
+            "summaries": shown,
+            "unusable": unusable,
+            "near_duplicates": collapsed,
+            "scanned": len(rows),
+        })
+    except sqlite3.Error as exc:
+        # A store that predates the display table. Empty, not an error.
+        logger.debug("consolidated summaries read failed: %s", exc)
+        return JSONResponse({
+            "profile": pid, "summaries": [], "unusable": 0, "scanned": 0,
+        })
+    finally:
+        conn.close()
+
+
+@router.get("/health")
+def get_memory_health() -> JSONResponse:
+    """Whether this store's memories can actually be found.
+
+    Same measurement ``slm doctor`` prints, so the dashboard and the CLI cannot
+    tell the owner two different things. Read-only and fail-soft.
+    """
+    try:
+        from superlocalmemory.core.memory_health import describe, measure
+
+        health = measure(DB_PATH)
+        return JSONResponse({
+            "live_facts": health.live_facts,
+            "findable_by_meaning": health.findable_by_meaning,
+            "missing_vector": health.missing_vector,
+            "withheld_summaries": health.withheld_summaries,
+            "display_summaries": health.display_summaries,
+            "hidden_by_forgetting": health.hidden_by_forgetting,
+            "inconsistently_hidden": health.inconsistently_hidden,
+            "reachability": round(health.reachability, 4),
+            "healthy": health.healthy,
+            "unavailable": list(health.unavailable),
+            "summary": describe(health),
+        })
+    except Exception as exc:  # pragma: no cover - defensive
+        logger.debug("memory health read failed: %s", exc)
+        return JSONResponse({"healthy": None, "summary": [], "unavailable": ["error"]})

package/src/superlocalmemory/server/routes/data_io.py

@@ -12,7 +12,7 @@ import gzip
 import hashlib
 import json
 import logging
-from typing import Optional
+from typing import Any, Optional
 from datetime import datetime, timezone
 
 from fastapi import APIRouter, HTTPException, Query, Request, UploadFile, File
@@ -78,7 +78,22 @@ async def export_memories(
             use_v3 = False
 
         if use_v3:
+            # Withheld summaries are excluded, and this is about IMPORT, not
+            # tidiness. Import re-ingests every record through the normal
+            # pipeline, which mints a fresh memory with quarantined = 0 — so an
+            # export taken here and imported anywhere would resurrect all 1,195
+            # model-written rows as though the owner had written them, on a
+            # machine where nothing had gone wrong. The repair would then have
+            # to run again there.
+            #
+            # Nothing the owner wrote is lost: these are derived artefacts, the
+            # consolidator regenerates them from the facts that ARE exported,
+            # and their text is kept in consolidated_summaries. (That table is
+            # not in this export either — it is a view, not a memory. Worth
+            # revisiting when export covers derived state.)
             query = "SELECT * FROM atomic_facts WHERE profile_id = ?"
+            if _has_column(cursor, "atomic_facts", "quarantined"):
+                query += " AND COALESCE(quarantined, 0) = 0"
             params = [active_profile]
             if category:
                 query += " AND fact_type = ?"
@@ -154,6 +169,19 @@ async def export_memories(
         raise _internal_error("Export error")
 
 
+def _has_column(cursor: Any, table: str, column: str) -> bool:
+    """Whether ``table`` carries ``column`` in this database.
+
+    Presence-guarded because ``quarantined`` arrives with a migration and this
+    route must keep working on a store the engine has not opened.
+    """
+    try:
+        cursor.execute(f"PRAGMA table_info({table})")
+        return any(row[1] == column for row in cursor.fetchall())
+    except Exception:  # noqa: BLE001 -- an export must not fail over a probe
+        return False
+
+
 @router.post("/api/import")
 async def import_memories(request: Request, file: UploadFile = File(...)):
     """Import memories from JSON file using V3 engine."""

package/src/superlocalmemory/server/routes/entity.py

@@ -84,7 +84,19 @@ def list_entities(
 
     conn = get_read_connection(engine._config.db_path)
     try:
-        where = ["ce.profile_id = ?"]
+        # Hide the placeholder that dated facts with no recognised entity
+        # attach their temporal events to (core/store_pipeline.py,
+        # _ensure_unresolved_entity). It is a hook for a foreign key, not a
+        # concept, and it would otherwise appear here as a nameless entity and
+        # be counted in the total the owner reads as "entities I have".
+        #
+        # Only the presentation layer needs this. The other readers of
+        # canonical_entities are unaffected on inspection: graph_pruner uses
+        # `NOT IN (SELECT entity_id ...)` to find orphaned edges, where
+        # including it is correct; community_summary builds from graph edges and
+        # the placeholder has none; scale_engine counts for backend sync, not
+        # for display.
+        where = ["ce.profile_id = ?", "ce.entity_type != 'unresolved'"]
         params: list[object] = [profile]
         if entity_type and entity_type.lower() != "all":
             where.append("ce.entity_type = ? COLLATE NOCASE")

package/src/superlocalmemory/server/routes/mesh.py

@@ -408,7 +408,7 @@ def send(req: SendRequest, request: Request):
         if sig_err is not None:
             raise HTTPException(401, detail=sig_err.get("error", "signature error"))
 
-        # Admission gate parity (closes Wave-1 P1 bypass for inbound remote send).
+        # Admission gate parity (closes the P1 bypass for inbound remote send).
         try:
             from superlocalmemory.core.admission import (
                 AdmissionDenied,

package/src/superlocalmemory/server/routes/v3_api.py

@@ -266,7 +266,7 @@ def apply_settings_update(config: "SLMConfig", payload: dict) -> "SLMConfig":
     This is the SINGLE authoritative place where incoming dashboard save payloads
     are merged onto the stored config.  The POST /api/v3/mode/set handler calls
     this function after its HTTP-layer concerns (SSRF guard, auth) are settled,
-    so the acceptance gate (test_wave2_acceptance.py::P3) tests real product
+    so the test suite (test_acceptance_core.py::P3) tests real product
     behaviour — not a reimplementation.
 
     SEC-L-01 PRESERVED:
@@ -444,7 +444,7 @@ async def set_full_config(request: Request):
         # Resolve the effective endpoint value before SSRF validation.
         # Only the Ollama default injection happens here; the fallback to the
         # stored URL (and redacted-echo detection) live inside
-        # apply_settings_update so that the P3 acceptance gate exercises
+        # apply_settings_update so that the P3 test exercises
         # the same code path as the HTTP handler — not a reimplementation.
         _raw_ep: str = "" if clear_base_url else (base_url_input or endpoint_input or "")
         if (

package/src/superlocalmemory/server/ui.py

@@ -231,7 +231,14 @@ def create_app() -> FastAPI:
                 "<p><a href='/api/docs'>API Documentation</a></p>"
                 "</body></html>"
             )
-        return index_path.read_text()
+        from superlocalmemory.server.asset_versions import render_index
+        from superlocalmemory import __version__ as _v
+
+        # __SLM_VERSION__ was substituted only by the unified daemon, so the
+        # dashboard's upgrade detector did nothing when served from here.
+        return render_index(
+            index_path, UI_DIR, substitutions={"__SLM_VERSION__": _v},
+        )
 
     @application.get("/favicon.ico", include_in_schema=False)
     async def favicon():

package/src/superlocalmemory/server/unified_daemon.py

@@ -52,7 +52,7 @@ os.environ.setdefault("SLM_MCP_EMBEDDED", "1")
 from fastapi import FastAPI, HTTPException, Request
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.middleware.gzip import GZipMiddleware
-from pydantic import BaseModel
+from pydantic import BaseModel, field_validator
 
 from superlocalmemory.core.config import CANONICAL_RECALL_LIMIT
 from superlocalmemory.infra.daemon_identity import (
@@ -587,6 +587,62 @@ class RememberRequest(BaseModel):
     metadata: dict | None = None  # v3.4.26: pass-through from MCP pool_store
     idempotency_key: str | None = None
     session_id: str = ""
+    #: WHEN this memory is about, as distinct from when it was written.
+    #:
+    #: The internal admission record has carried this field all along and this
+    #: model never had it, so every memory arriving over HTTP — which is every
+    #: memory, from the CLI, the tool interface and the dashboard alike — was
+    #: stamped with its ingestion date. Measured on the author's store: 200 of
+    #: the 200 most recent facts have an observation_date, and 196 of them are
+    #: the day they were written. A store that cannot be told "this happened in
+    #: March" cannot answer a question about March.
+    #:
+    #: Empty means "today", the previous behaviour. Format is YYYY-MM-DD or a
+    #: full ISO 8601 timestamp.
+    session_date: str = ""
+
+    @field_validator("session_date")
+    @classmethod
+    def _session_date_is_a_date(cls, value: str) -> str:
+        """Reject a malformed date rather than ignore it.
+
+        Dropping it silently would leave the caller believing the date was
+        recorded while the memory quietly filed itself under today — which is
+        the exact failure this field exists to fix, reintroduced one layer up.
+        Rejecting is recoverable: the caller sees the error and resends.
+
+        ISO ONLY, DELIBERATELY, even though this system ships a parser that is
+        far more permissive. ``encoding/temporal_parser.parse_session_date``
+        accepts "May 8, 2026", "1:56 pm on 8 May, 2026", "14/03/2026", and also
+        "last tuesday" and "March 2026" — and those last two are why it is not
+        used here. Asked on 2026-08-21 it resolves "last tuesday" to
+        **2026-08-25**, four days into the future, and "March 2026" to the 21st,
+        a day it invents. Accepting either at this boundary would file a memory
+        under a confidently wrong date, silently, which is precisely the class
+        of defect this field was added to remove.
+
+        The caller here is a program — the command line, the tool interface, the
+        dashboard — not a person typing. ISO is the right contract for a
+        program, and a caller holding a human-typed date can run it through the
+        parser itself and send the result.
+        """
+        if not value:
+            return value
+        from datetime import datetime as _dt
+
+        text = value.strip()
+        try:
+            _dt.fromisoformat(text.replace("Z", "+00:00"))
+        except ValueError:
+            raise ValueError(
+                "session_date must be YYYY-MM-DD or a full ISO 8601 timestamp; "
+                f"got {value!r}. To accept looser human phrasing, parse it "
+                "first with TemporalParser.parse_session_date and send the ISO "
+                "result — but check what it returns, because it resolves "
+                "relative phrases against today and can answer with a future "
+                "date."
+            ) from None
+        return text
     # v3.6.15 multi-scope: visibility of the new memory. ``None`` scope means
     # "use the configured default_scope" (personal). shared_with is the list of
     # profile_ids for scope='shared'.
@@ -2399,6 +2455,10 @@ async def lifespan(application: FastAPI):
                 "state": "checking_components", "embeddings_backfilled": 0,
                 "expansion_backfilled": 0, "null_remaining": None,
                 "components": None,
+                # Declared here so /status carries the same keys whatever
+                # happens; a field that only appears on failure is a field
+                # nobody's dashboard renders.
+                "incomplete_reason": None,
                 "started_at": _t.time(), "finished_at": None,
             }
             # Step 0 (v3.8.2 "whole self-healer"): repair components that
@@ -2507,9 +2567,42 @@ async def lifespan(application: FastAPI):
                     _backfill_vector_store()
                 except Exception as exc:
                     logger.warning("Self-heal vector index failed (non-fatal): %s", exc)
-                _SELF_HEAL_STATUS["state"] = "complete"
+
+                # "complete" has to mean complete.
+                #
+                # This line used to set "complete" unconditionally, so a
+                # backfill that gave up after five no-progress attempts, or ran
+                # out its 500-iteration budget, reported success with facts
+                # still unembedded — and an unembedded fact cannot be found by
+                # meaning at all. That is how a machine sat at 56.3% of its
+                # memory reachable while its own status endpoint said the heal
+                # had finished. Nobody was going to look past a green light.
+                #
+                # Now the state is derived from the remaining count, and the
+                # gap is named in plain language for the dashboard, so an
+                # incomplete heal is visible and gets retried on next start
+                # instead of being declared done forever.
+                _remaining = _SELF_HEAL_STATUS.get("null_remaining")
                 _SELF_HEAL_STATUS["finished_at"] = _t.time()
-                logger.info("Self-heal complete: %s", _SELF_HEAL_STATUS)
+                if _remaining is None:
+                    _SELF_HEAL_STATUS["state"] = "complete"
+                elif int(_remaining) > 0:
+                    _SELF_HEAL_STATUS["state"] = "incomplete"
+                    _SELF_HEAL_STATUS["incomplete_reason"] = (
+                        f"{int(_remaining)} memories still have no meaning "
+                        f"vector, so they cannot be found by asking a question. "
+                        f"This retries automatically next time the service "
+                        f"starts. It usually means the embedding model was "
+                        f"unavailable."
+                    )
+                    logger.warning(
+                        "Self-heal INCOMPLETE: %d facts still unembedded and "
+                        "therefore unreachable by meaning; will retry on next "
+                        "start. %s", int(_remaining), _SELF_HEAL_STATUS,
+                    )
+                else:
+                    _SELF_HEAL_STATUS["state"] = "complete"
+                    logger.info("Self-heal complete: %s", _SELF_HEAL_STATUS)
             except Exception as exc:
                 _SELF_HEAL_STATUS["state"] = "error"
                 logger.warning("Self-heal failed (non-fatal): %s", exc)
@@ -3975,8 +4068,16 @@ def _register_dashboard_routes(application: FastAPI) -> None:
         # v3.4.23: substitute version placeholder so the dashboard can detect
         # upgrades and auto-reload. Read fresh each request (daemon uptime is
         # days, but we want zero caching surprises during development).
-        html = index_path.read_text()
-        return html.replace("__SLM_VERSION__", _SLM_VERSION)
+        #
+        # 4.0.10: asset ?v= strings are now derived from file content instead of
+        # being hand-written literals that tracked nothing. See
+        # server/asset_versions.py — including what that does and does not fix.
+        from superlocalmemory.server.asset_versions import render_index
+
+        return render_index(
+            index_path, UI_DIR,
+            substitutions={"__SLM_VERSION__": _SLM_VERSION},
+        )
 
     @application.get("/favicon.ico", include_in_schema=False)
     async def favicon():
@@ -4146,7 +4247,7 @@ def _register_daemon_routes(application: FastAPI) -> None:
             "runtime_state": runtime_state,
             "active_profile": profile_snapshot.profile_id,
             "profile_generation": profile_snapshot.generation,
-            # Wave-3: operational failure counts (visible to all team members)
+            # operational failure counts (visible to all team members)
             **_ops_failure_counts(engine, application),
             # issue #107: does this daemon's *imported* code still match the
             # installed distribution? ``version`` above reports what this
@@ -4513,6 +4614,7 @@ def _register_daemon_routes(application: FastAPI) -> None:
                 shared_with=tuple(shared_with or ()),
                 trusted_actor_id=trusted_actor_id,
                 session_id=req.session_id,
+                session_date=req.session_date,
             )
             actor = Actor(
                 principal_id=trusted_actor_id,
@@ -4859,7 +4961,7 @@ def _register_daemon_routes(application: FastAPI) -> None:
             # index backfill after an upgrade). Dashboard renders a plain
             # "Optimizing memory…" line from this. Defaults to idle before start.
             "self_heal": globals().get("_SELF_HEAL_STATUS", {"state": "idle"}),
-            # Wave-3: operational failure counts (dead-letter, degraded, stalled)
+            # operational failure counts (dead-letter, degraded, stalled)
             **_ops_failure_counts(engine, application),
         }
 
@@ -4913,7 +5015,7 @@ def _register_daemon_routes(application: FastAPI) -> None:
         return {"status": "started"}
 
     # ------------------------------------------------------------------
-    # Wave-3: Operational Recovery & Admin Remediation  (V4 resilience slice)
+    # Operational Recovery & Admin Remediation  (V4 resilience slice)
     # ------------------------------------------------------------------
 
     @application.get("/operations/failed")
@@ -5460,7 +5562,7 @@ def _terminalize_orphan_operation(engine, operation_id: str) -> None:
 
 
 def _ops_failure_counts(engine, application) -> dict:
-    """Return Wave-3 operational failure counts for /status and /health.
+    """Return operational failure counts for /status and /health.
 
     Always returns a dict (never raises). Counts default to 0 on any error.
     Includes: dead_letter_count, degraded_operations, exhausted_obligations,

package/src/superlocalmemory/storage/_migration_internals.py

@@ -153,6 +153,9 @@ from superlocalmemory.storage.migrations import (
 from superlocalmemory.storage.migrations import (
     M042_correction_case_ledger as _M042,
 )
+from superlocalmemory.storage.migrations import (
+    M043_quarantine_display_summaries as _M043,
+)
 
 # Emit under the runner's logger name so operational log filters that key on
 # "superlocalmemory.storage.migration_runner" keep matching after this split.
@@ -203,6 +206,7 @@ _MODULES = {
     _M040.NAME: _M040,
     _M041.NAME: _M041,
     _M042.NAME: _M042,
+    _M043.NAME: _M043,
 }
 
 # Exact historical DDL fingerprints whose resulting schema is intentionally