superlocalmemory 4.0.9 → 4.0.10

package/src/superlocalmemory/storage/database.py

@@ -686,8 +686,12 @@ class DatabaseManager:
             include_shared=include_shared,
             prefix="f",
         )
+        # Pins are injected straight into an agent's context, which makes this
+        # the most consequential display path in the class: a withheld row here
+        # is not merely shown, it is asserted as background truth.
         rows = self.execute(
             f"SELECT f.* FROM atomic_facts f WHERE {where} AND f.pinned = 1 "
+            f"{self.visible_fact_clause('f')} "
             "AND NOT EXISTS ("
             "    SELECT 1 FROM fact_temporal_validity tv "
             "    WHERE tv.fact_id = f.fact_id "
@@ -733,6 +737,65 @@ class DatabaseManager:
             self._archive_col_present = True
         return present
 
+    def _has_quarantine_column(self) -> bool:
+        """Whether atomic_facts carries the ``quarantined`` column.
+
+        Same shape as ``_has_archive_status``: cached once True (a column never
+        disappears), re-checked while absent so a later schema pass is picked
+        up. ``storage.schema.create_all_tables`` adds the column at every engine
+        init, so on any store the daemon has opened this is True — the guard
+        exists for a bare DatabaseManager pointed at a store that engine init
+        never touched, where filtering on the column would raise instead of
+        returning results.
+        """
+        if getattr(self, "_quarantine_col_present", False):
+            return True
+        present = any(
+            dict(row).get("name") == "quarantined"
+            for row in self.execute("PRAGMA table_info(atomic_facts)")
+        )
+        if present:
+            self._quarantine_col_present = True
+        return present
+
+    def visible_fact_clause(
+        self, prefix: str = "", *, include_quarantined: bool = False,
+    ) -> str:
+        """AND-clause excluding rows no caller should be shown as a memory.
+
+        Two exclusions, one definition: soft-deleted (``archive_status``) and
+        withheld (``quarantined``). Both are presence-guarded, because each
+        column arrives with a migration and may be absent on a store the engine
+        has not opened.
+
+        WHY THIS EXISTS AS A FUNCTION. 4.0.10 first put the quarantine filter in
+        ``get_facts_by_ids`` alone, reasoning that every retrieval channel
+        re-authorises through it and the engine drops what it cannot hydrate.
+        That reasoning was correct and the conclusion was wrong: it covered the
+        RECALL pipeline, and ``search``, ``list_recent``, ``fetch``, the MCP
+        resources and the dashboard's own search are not the recall pipeline.
+        Measured on a copy of the author's store, ``search_facts_fts`` returned
+        20 withheld rows out of 50 and ``get_all_facts`` 66 out of 400 — the
+        exact defect the design was meant to prevent, in the paths the design
+        never looked at.
+
+        There is no single SQL choke point in this codebase; ``_scope_where`` is
+        spliced against six other tables and cannot carry a fact column. So the
+        honest form of "one place" is one CLAUSE with an enumerable set of call
+        sites, and a test that fails when a read path does not use it:
+        tests/test_storage/test_no_read_path_shows_a_withheld_row.py
+
+        ``include_quarantined=True`` is for repair, erasure and export — paths
+        that must reach a withheld row to act on it.
+        """
+        table = f"{prefix}." if prefix else ""
+        clause = ""
+        if self._has_archive_status():
+            clause += f" AND COALESCE({table}archive_status, 'live') != 'archived'"
+        if not include_quarantined and self._has_quarantine_column():
+            clause += f" AND COALESCE({table}quarantined, 0) = 0"
+        return clause
+
     def get_all_facts(
         self, profile_id: str, limit: int | None = None,
         *,
@@ -755,14 +818,10 @@ class DatabaseManager:
         # hard, env-tunable ceiling even when the caller passes limit=None.
         if limit is None:
             limit = _unbounded_facts_ceiling()
-        # Archived facts are not live; never surface them in direct reads.
-        archive_clause = (
-            " AND COALESCE(archive_status, 'live') != 'archived'"
-            if self._has_archive_status()
-            else ""
-        )
+        # Soft-deleted and withheld rows are not memories a caller may see.
         rows = self.execute(
-            f"SELECT * FROM atomic_facts WHERE {where}{archive_clause} "
+            f"SELECT * FROM atomic_facts WHERE {where}"
+            f"{self.visible_fact_clause()} "
             "ORDER BY created_at DESC LIMIT ?",
             (*params, int(limit)),
         )
@@ -790,14 +849,12 @@ class DatabaseManager:
             include_global=include_global,
             include_shared=include_shared,
         )
-        archive_clause = (
-            " AND COALESCE(archive_status, 'live') != 'archived'"
-            if self._has_archive_status()
-            else ""
-        )
+        # Crossing a profile boundary is the last place a withheld row should
+        # appear: it would be a model's non-answer presented to somebody else
+        # as one of this profile's shared memories.
         rows = self.execute(
             f"SELECT * FROM atomic_facts WHERE {where} AND profile_id != ?"
-            f"{archive_clause} ORDER BY created_at DESC",
+            f"{self.visible_fact_clause()} ORDER BY created_at DESC",
             (*params, profile_id),
         )
         return [self._row_to_fact(r) for r in rows]
@@ -953,14 +1010,21 @@ class DatabaseManager:
         include_global: bool = False,
         include_shared: bool = False,
     ) -> int:
-        """Total fact count for a profile."""
+        """Memories this profile has, as the owner would count them.
+
+        Counts what a caller can be shown, which is why it applies
+        ``visible_fact_clause``. It fed the dashboard's "All memories 5,093" and
+        was counting 1,195 withheld summaries and every soft-deleted row into
+        that figure -- a number the owner reads as "how much do I remember".
+        """
         where, params = _scope_where(
             profile_id,
             include_global=include_global,
             include_shared=include_shared,
         )
         rows = self.execute(
-            f"SELECT COUNT(*) AS c FROM atomic_facts WHERE {where}", (*params,),
+            f"SELECT COUNT(*) AS c FROM atomic_facts WHERE {where}"
+            f"{self.visible_fact_clause()}", (*params,),
         )
         return int(rows[0]["c"]) if rows else 0
 
@@ -1230,16 +1294,15 @@ class DatabaseManager:
             include_shared=include_shared,
             prefix="f",
         )
-        # Archived facts must not surface via full-text search either.
-        archive_clause = (
-            " AND COALESCE(f.archive_status, 'live') != 'archived'"
-            if self._has_archive_status()
-            else ""
-        )
+        # Full-text search is a display path: the dashboard search box, the
+        # `search` tool and `fetch` all land here, and none of them go through
+        # the recall engine. Before 4.0.10 put the clause here it returned 20
+        # withheld rows out of 50 on the author's store.
         rows = self.execute(
             f"""SELECT f.* FROM atomic_facts_fts AS fts
                JOIN atomic_facts AS f ON f.fact_id = fts.fact_id
-               WHERE fts.atomic_facts_fts MATCH ? AND {where}{archive_clause}
+               WHERE fts.atomic_facts_fts MATCH ? AND {where}
+                     {self.visible_fact_clause('f')}
                ORDER BY fts.rank LIMIT ?""",
             (match_expr, *params, limit),
         )
@@ -1270,12 +1333,22 @@ class DatabaseManager:
     # ------------------------------------------------------------------
 
     def get_fact(self, fact_id: str, profile_id: str | None = None) -> AtomicFact | None:
-        """Get a single fact by ID.
+        """Get a single row by ID, exactly as stored. NOT a display path.
 
         C4 defense-in-depth: when ``profile_id`` is provided the lookup is
         tenant-scoped so a fact_id from another profile cannot resolve. Left
         optional (fact_id is a random UUID sourced from already-scoped queries)
         to avoid destabilizing the core store/consolidation write path.
+
+        DELIBERATELY UNFILTERED, and this is load-bearing. It applies neither
+        ``archive_status`` nor ``quarantined`` because it is the primitive that
+        write paths, correction handling and the 4.0.10 repair use to read a row
+        they already hold the id of — including a withheld one, which they must
+        be able to see in order to act on it. ``visible_fact_clause`` is for the
+        paths that answer a question; this one answers "what is in that row".
+
+        A caller taking a fact_id from user input and rendering the result wants
+        ``get_facts_by_ids`` instead.
         """
         if profile_id is not None:
             rows = self.execute(
@@ -1292,8 +1365,36 @@ class DatabaseManager:
         self, fact_ids: list[str], profile_id: str,
         include_global: bool = False,
         include_shared: bool = False,
+        *,
+        include_quarantined: bool = False,
     ) -> list[AtomicFact]:
-        """Get multiple facts by their IDs, scoped to a profile."""
+        """Get multiple facts by their IDs, scoped to a profile.
+
+        THIS IS THE PLACE QUARANTINE IS ENFORCED, and the only one.
+
+        Every retrieval channel re-authorises its candidates through here
+        (``retrieval/scope_policy.py`` — "candidate generators may use caches,
+        approximate indexes, or graph stores that are not the authorization
+        source of truth"), and the engine hydrates the fused set from here too.
+        A fact this method does not return has no content to show, and
+        ``retrieval/engine.py`` drops it: ``if fact is None: continue``. So one
+        clause here covers bm25, semantic, temporal, entity, hopfield and
+        spreading activation, in normal and deep recall alike, whether or not
+        the forgetting filter is registered.
+
+        The alternatives were checked and rejected. ``_scope_where`` looks like
+        the natural home but is spliced against ``graph_edges``,
+        ``temporal_events``, ``memories``, ``bm25_tokens``,
+        ``fact_temporal_validity`` and ``correction_cases`` as well as
+        ``atomic_facts``, so a column reference there breaks eight call sites.
+        ``ForgettingFilter`` is optional (it no-ops when forgetting is
+        disabled) and excludes nothing in deep recall.
+
+        ``include_quarantined=True`` is for repair, export and erasure — paths
+        that must be able to see a withheld row in order to act on it. It is
+        keyword-only and greppable on purpose: every caller that opts in is
+        meant to be found in one search.
+        """
         if not fact_ids:
             return []
         where, params = _scope_where(
@@ -1301,15 +1402,12 @@ class DatabaseManager:
             include_global=include_global,
             include_shared=include_shared,
         )
-        archive_clause = (
-            " AND COALESCE(archive_status, 'live') != 'archived'"
-            if self._has_archive_status()
-            else ""
-        )
         placeholders = ",".join("?" for _ in fact_ids)
         rows = self.execute(
             f"SELECT * FROM atomic_facts WHERE fact_id IN ({placeholders}) "
-            f"AND {where}{archive_clause} ORDER BY created_at DESC",
+            f"AND {where}"
+            f"{self.visible_fact_clause(include_quarantined=include_quarantined)} "
+            "ORDER BY created_at DESC",
             (*fact_ids, *params),
         )
         return [self._row_to_fact(r) for r in rows]

package/src/superlocalmemory/storage/migration_runner.py

@@ -159,6 +159,9 @@ from superlocalmemory.storage.migrations import (
 from superlocalmemory.storage.migrations import (
     M042_correction_case_ledger as _M042,
 )
+from superlocalmemory.storage.migrations import (
+    M043_quarantine_display_summaries as _M043,
+)
 from superlocalmemory.storage._schema_version import (
     SUPPORTED_SCHEMA_VERSION,
     SchemaVersionError,
@@ -312,6 +315,14 @@ DEFERRED_MIGRATIONS: list[Migration] = [
     # Main-line M034 is renumbered in V4. It must remain deferred because its
     # backfill joins engine-bootstrapped memory_scenes and atomic_facts.
     Migration(name=_M039.NAME, db_target="memory", ddl=_M039.DDL),
+    # M043 withholds model-written summaries from the retrieval corpus and
+    # un-hides the memories they displaced. Deferred because it reads and
+    # writes atomic_facts + fact_retention, both bootstrapped at engine init —
+    # the same reason M011/M013/M015/M016 are deferred. apply_deferred takes a
+    # verified snapshot before the first migration it actually applies, so the
+    # store is recoverable.
+    Migration(name=_M043.NAME, db_target="memory", ddl=_M043.DDL,
+              dependencies=(_M011.NAME,)),
 ]