superlocalmemory 4.0.9 → 4.1.0

package/src/superlocalmemory/core/graph_pruner.py

@@ -191,17 +191,138 @@ def _batch_delete_by_ids(
     for start in range(0, len(ids), _BATCH_SIZE):
         batch = ids[start:start + _BATCH_SIZE]
         ph = ",".join("?" * len(batch))
+        endpoints = _endpoints_of(db, table, id_col, batch)
         with db.transaction():
             db.execute(
                 f"DELETE FROM {table} WHERE {id_col} IN ({ph})",
                 tuple(batch),
             )
+            # Inside the same transaction as the delete. The graph also lives in
+            # a second store, which no SQLite transaction can reach, so the
+            # durable record of "these facts need re-projecting" has to be as
+            # durable as the delete itself. Without it the second store keeps
+            # serving edges this function just removed, and a search walks a hop
+            # that no longer exists.
+            _enqueue_reprojection(db, endpoints)
         removed += len(batch)
         if start + _BATCH_SIZE < len(ids):
             time.sleep(_BATCH_YIELD_S)
     return removed
 
 
+#: The two columns naming an edge's ends, per table. ``association_edges``
+#: calls them ``source_fact_id``/``target_fact_id``, so a helper that assumed
+#: ``source_id``/``target_id`` raised there, returned nothing, and every
+#: association-edge deletion went unannounced to the graph store -- silently,
+#: because the failure looked exactly like "this table has no endpoints".
+_ENDPOINT_COLUMNS: dict[str, tuple[str, str]] = {
+    "graph_edges": ("source_id", "target_id"),
+    "association_edges": ("source_fact_id", "target_fact_id"),
+}
+
+
+def _endpoints_of(
+    db: "DatabaseManager", table: str, id_col: str, ids: list,
+) -> list[tuple[str, str]]:
+    """``(fact_id, profile_id)`` for both ends of the rows about to be deleted.
+
+    Read before the delete, because afterwards there is nothing to read. Only
+    the ends that are facts matter: an entity id here is projected as part of
+    whichever facts reference it, and those facts are re-derived anyway.
+    """
+    if not ids:
+        return []
+    columns = _ENDPOINT_COLUMNS.get(table)
+    if columns is None:
+        # A table nobody has named the endpoints of is one this pass must not
+        # guess at. Loudly, because guessing wrong is what produced the silent
+        # hole above.
+        logger.warning(
+            "prune: %s has no declared endpoint columns, so its deletions "
+            "cannot be announced to the graph projection", table,
+        )
+        return []
+    ph = ",".join("?" * len(ids))
+    try:
+        rows = db.execute(
+            f"SELECT {columns[0]}, {columns[1]}, profile_id FROM {table} "
+            f"WHERE {id_col} IN ({ph})",
+            tuple(ids),
+        )
+    except Exception as exc:  # noqa: BLE001
+        logger.warning("prune: cannot read endpoints from %s: %s", table, exc)
+        return []
+    seen: dict[tuple[str, str], None] = {}
+    for row in rows:
+        record = dict(row)
+        profile_id = str(record.get("profile_id") or "default")
+        for column in columns:
+            value = record.get(column)
+            if value:
+                seen[(str(value), profile_id)] = None
+    return list(seen)
+
+
+def _enqueue_reprojection(
+    db: "DatabaseManager", endpoints: list[tuple[str, str]],
+) -> None:
+    """Queue an upsert for each fact whose edges just changed.
+
+    The queue coalesces on fact id and the worker re-reads the fact's current
+    edges from SQLite, so queueing an endpoint twice, or queueing one whose
+    edges were already correct, costs one row and converges on the same answer.
+    An id that is an entity rather than a fact is filtered out here rather than
+    left for the worker, which would otherwise spend a lookup discovering the
+    same thing on every cycle.
+    """
+    if not endpoints:
+        return
+    try:
+        from superlocalmemory.storage import projection_outbox
+    except Exception as exc:  # noqa: BLE001
+        logger.debug("prune: no projection queue module: %s", exc)
+        return
+    try:
+        if not projection_outbox.is_available(db):
+            return
+    except Exception as exc:  # noqa: BLE001
+        logger.debug("prune: projection queue unavailable: %s", exc)
+        return
+    by_profile: dict[str, list[str]] = {}
+    for fact_id, profile_id in endpoints:
+        by_profile.setdefault(profile_id, []).append(fact_id)
+    for profile_id, fact_ids in by_profile.items():
+        ph = ",".join("?" * len(fact_ids))
+        try:
+            rows = db.execute(
+                f"SELECT fact_id FROM atomic_facts WHERE profile_id = ? "
+                f"AND fact_id IN ({ph})",
+                (profile_id, *fact_ids),
+            )
+            real = [dict(row)["fact_id"] for row in rows]
+        except Exception as exc:  # noqa: BLE001
+            logger.debug("prune: cannot confirm endpoints are facts: %s", exc)
+            continue
+        if not real:
+            continue
+        # Deliberately allowed to raise. This runs inside the same transaction
+        # as the delete, so a failure here rolls the delete back and the batch
+        # is simply retried on the next pass -- nothing is lost and nothing
+        # diverges.
+        #
+        # Swallowing it, as this once did, committed the delete with no record
+        # that the graph needed telling. The queue would then be empty, which
+        # is exactly what "the graph is up to date" looks like, and the graph
+        # would serve a link the store had removed with nothing anywhere
+        # recording that it happened. That is the failure this queue exists to
+        # make impossible, and it is the module's own stated policy: "a
+        # durability mechanism that silently degrades to best-effort is the
+        # defect it exists to remove."
+        projection_outbox.enqueue_many(
+            db, real, profile_id, op=projection_outbox.OP_UPSERT,
+        )
+
+
 def _remove_orphan_edges_batched(
     db: "DatabaseManager",
     profile_id: str,

package/src/superlocalmemory/core/maintenance.py

@@ -31,6 +31,16 @@ if TYPE_CHECKING:
 
 logger = logging.getLogger(__name__)
 
+
+class _ConsolidationDisabled(Exception):
+    """Internal signal: consolidation is switched off, so skip its block.
+
+    A private exception rather than restructuring the surrounding try/except:
+    the block's job is to keep one optional maintenance step from taking the
+    whole pass down with it, and that guarantee should not be weakened to
+    express "deliberately skipped". Caught immediately below, never propagated.
+    """
+
 # Backfill constants
 _BACKFILL_BURN_IN_STEPS = 50
 _LANGEVIN_DIM = 8
@@ -638,9 +648,12 @@ def run_maintenance(
         logger.warning("Entity summary consolidation failed: %s", exc)
 
     # 4. Fact consolidation (v3.8.4 concurrency-safe path via DatabaseManager).
-    # Merges clusters of warm/cold atomic facts about the same entity into a
-    # single consolidated fact, archives the originals (NEVER deletes them),
-    # and records provenance in fact_consolidations.
+    # Groups warm/cold atomic facts that share an entity and writes ONE
+    # DISPLAY summary per cluster into consolidated_summaries, with provenance
+    # in fact_consolidations. It does not write to atomic_facts and does not
+    # archive the source facts — until 4.0.10 it did both, which put 1,195
+    # model-written rows into the retrieval corpus and left 528 genuine
+    # memories archived out of normal recall.
     #
     # Uses the DatabaseManager path so LLM calls happen OUTSIDE the write lock:
     #   - Discover clusters in a short memory_read() (no write lock held).
@@ -652,20 +665,45 @@ def run_maintenance(
     try:
         from superlocalmemory.core.fact_consolidator import consolidate_facts
 
+        # The documented off-switch has to actually switch something off.
+        # ConsolidationConfig.enabled has existed since Phase 5 and this call
+        # site never read it, so a user who ran `slm config` to turn
+        # consolidation off got consolidation anyway — for four months, on
+        # every maintenance pass. -2 is a third distinguishable value, kept
+        # apart from 0 (nothing to merge) and -1 (the step failed), so a
+        # deliberately disabled step is never mistaken for either.
+        _consolidation = getattr(config, "consolidation", None)
+        if _consolidation is not None and not getattr(_consolidation, "enabled", True):
+            counts["facts_consolidated"] = -2
+            logger.debug("Fact consolidation disabled by configuration")
+            raise _ConsolidationDisabled
+
         fc_stats = consolidate_facts(
             db,
             profile_id=profile_id,
-            max_clusters=getattr(config, "max_consolidation_clusters", 20),
+            # Read from ConsolidationConfig, with the old SLMConfig-level name
+            # as the fallback. `getattr(config, "max_consolidation_clusters")`
+            # alone never resolved — SLMConfig has no such attribute — so the
+            # default was the only value this had ever used.
+            max_clusters=int(
+                getattr(_consolidation, "max_consolidation_clusters", None)
+                or getattr(config, "max_consolidation_clusters", None)
+                or 20
+            ),
             dry_run=False,
             config=config,
         )
         counts["facts_consolidated"] = fc_stats.get("consolidated", 0)
         if fc_stats.get("consolidated", 0) > 0:
             logger.info(
-                "Fact consolidation: %d clusters merged, %d facts archived",
+                "Fact consolidation: %d display summaries over %d facts "
+                "(%d clusters refused)",
                 fc_stats.get("consolidated", 0),
-                fc_stats.get("facts_archived", 0),
+                fc_stats.get("facts_summarized", 0),
+                fc_stats.get("rejected", 0),
             )
+    except _ConsolidationDisabled:
+        pass
     except Exception as exc:
         # WARNING, not debug, and a distinguishable count. Leaving this at debug
         # with facts_consolidated=0 made a failing consolidation report exactly

package/src/superlocalmemory/core/maintenance_scheduler.py

@@ -67,6 +67,16 @@ class MaintenanceScheduler:
         self._initial_gc_timer = threading.Timer(90.0, self._initial_cache_gc)
         self._initial_gc_timer.daemon = True
         self._initial_gc_timer.start()
+        # An upgrade arrives with whatever backlog the previous version left.
+        # Waiting a full interval for the first graph-metrics pass would mean
+        # half an hour of ranking memories as though they had no position in the
+        # graph, on exactly the store that just gained the fix. Staggered behind
+        # the cache GC so the two never contend for the write lock.
+        self._initial_metrics_timer = threading.Timer(
+            150.0, self._initial_graph_metrics,
+        )
+        self._initial_metrics_timer.daemon = True
+        self._initial_metrics_timer.start()
         logger.info(
             "Maintenance scheduler started (interval=%dm)",
             self._config.forgetting.scheduler_interval_minutes,
@@ -86,6 +96,29 @@ class MaintenanceScheduler:
         except Exception as exc:
             logger.debug("Startup activation-cache GC skipped: %s", exc)
 
+    def _initial_graph_metrics(self) -> None:
+        """One-shot catch-up so an upgrade does not rank on stale metrics."""
+        if not self._running:
+            return
+        try:
+            from superlocalmemory.core.graph_metrics import (
+                compute_graph_metrics,
+                metrics_are_stale,
+            )
+            for profile_id in self._profile_ids():
+                stale, why = metrics_are_stale(self._db, profile_id)
+                if not stale:
+                    continue
+                report = compute_graph_metrics(self._db, profile_id)
+                if report.ok:
+                    logger.info(
+                        "Graph metrics at startup (%s): %s", why, report.summary(),
+                    )
+                else:
+                    logger.warning("Graph metrics at startup: %s", report.summary())
+        except Exception as exc:
+            logger.debug("Startup graph metrics skipped: %s", exc)
+
     def stop(self) -> None:
         """Stop the scheduler. Idempotent."""
         self._running = False
@@ -96,6 +129,10 @@ class MaintenanceScheduler:
         if _gc_timer is not None:
             _gc_timer.cancel()
             self._initial_gc_timer = None
+        _metrics_timer = getattr(self, "_initial_metrics_timer", None)
+        if _metrics_timer is not None:
+            _metrics_timer.cancel()
+            self._initial_metrics_timer = None
         logger.info("Maintenance scheduler stopped")
 
     def _schedule_next(self) -> None:
@@ -106,6 +143,47 @@ class MaintenanceScheduler:
         self._timer.daemon = True
         self._timer.start()
 
+    #: Consecutive failures of one step before it stops being a hiccup.
+    _ESCALATE_AFTER = 3
+
+    def _record_step(self, step: str, ok: bool, detail: str = "") -> None:
+        """Remember whether a maintenance step worked, and say so when it has
+        stopped working."""
+        counts = getattr(self, "_step_failures", None)
+        if counts is None:
+            counts = self._step_failures = {}
+        if ok:
+            if counts.pop(step, 0):
+                logger.info("maintenance: %s is working again", step)
+            return
+        counts[step] = counts.get(step, 0) + 1
+        if counts[step] >= self._ESCALATE_AFTER:
+            logger.error(
+                "maintenance: %s has failed %d cycles in a row (%s). This is "
+                "not a transient failure; the work it does is not being done.",
+                step, counts[step], detail or "no detail",
+            )
+        else:
+            logger.warning("maintenance: %s failed (%s)", step, detail or "")
+
+    def failing_steps(self) -> dict[str, int]:
+        """Steps that have failed on consecutive cycles, and how many.
+
+        Read by the status surfaces, so an operator can see a persistently
+        broken maintenance step instead of having to find it in the log.
+        """
+        return dict(getattr(self, "_step_failures", {}) or {})
+
+    def _note_step_outcomes(self) -> None:
+        """Escalate anything still failing after this cycle's steps."""
+        failing = self.failing_steps()
+        if failing:
+            logger.warning(
+                "maintenance: %d step(s) still failing: %s",
+                len(failing),
+                ", ".join(f"{name} x{count}" for name, count in sorted(failing.items())),
+            )
+
     def _run(self) -> None:
         """Execute maintenance + auto-backup check, then schedule next run."""
         if not self._running:
@@ -183,6 +261,66 @@ class MaintenanceScheduler:
             except Exception as exc:
                 logger.debug("Graph pruning skipped for %s: %s", profile_id, exc)
 
+            # Pruning the graph orphans the lineage of every edge it removed,
+            # and nothing had ever deleted from that table — on a real store it
+            # had grown to 39% rows describing edges that no longer existed.
+            # This runs immediately after so the rows the pass just orphaned are
+            # collected in the same pass.
+            try:
+                from superlocalmemory.storage.lineage_retention import (
+                    prune_orphan_lineage,
+                )
+                report = prune_orphan_lineage(self._db, profile_id=profile_id)
+                if report.total:
+                    logger.info(
+                        "Lineage retention for %s: %d row(s) removed (%s)",
+                        profile_id, report.total, report.deleted,
+                    )
+            except Exception as exc:
+                logger.debug("Lineage retention skipped for %s: %s", profile_id, exc)
+
+            # Structural metrics. Recall multiplies a candidate's activation by
+            # its PageRank at every hop and biases it toward its query seeds'
+            # communities, and both numbers live in fact_importance -- so a
+            # memory missing from that table is found by the walk and then
+            # ranked as though it had no position in the graph.
+            #
+            # Nothing scheduled this. It ran only when a consolidation happened
+            # to fire or someone called the HTTP endpoint by hand, and on the
+            # author's store that meant one run in nine days: 1,036 of 4,034
+            # visible memories had no score and no community, and the newest
+            # four days of memories had none at all. This runs after pruning so
+            # it describes the graph that pruning left behind.
+            try:
+                from superlocalmemory.core.graph_metrics import (
+                    compute_graph_metrics,
+                    metrics_are_stale,
+                )
+                stale, why = metrics_are_stale(self._db, profile_id)
+                if stale:
+                    backend = None
+                    try:
+                        from superlocalmemory.core.backend_orchestrator import (
+                            get_orchestrator,
+                        )
+                        orchestrator = get_orchestrator()
+                        if orchestrator is not None:
+                            backend = orchestrator.get_graph_backend()
+                    except Exception:  # noqa: BLE001 -- in-process is the default anyway
+                        backend = None
+                    report = compute_graph_metrics(
+                        self._db, profile_id, backend=backend,
+                    )
+                    if report.ok:
+                        logger.info("Graph metrics (%s): %s", why, report.summary())
+                    else:
+                        logger.warning("Graph metrics: %s", report.summary())
+                else:
+                    logger.debug("Graph metrics up to date for %s", profile_id)
+                self._record_step("graph metrics", True)
+            except Exception as exc:  # noqa: BLE001
+                self._record_step("graph metrics", False, str(exc))
+
             # Lifecycle evaluation must cover every stored profile, not only
             # whichever profile was active when the engine started.
             try:
@@ -201,6 +339,73 @@ class MaintenanceScheduler:
             except Exception as exc:
                 logger.debug("Core-block recompile skipped for %s: %s", profile_id, exc)
 
+        # Re-read what is filed as a plan. The one-time pass runs as a
+        # migration; the rule it uses keeps getting sharper, and a completed
+        # migration is never replayed — so without this the store drifts
+        # further from the rule with every release and nothing repairs it. The
+        # pass is a pure function of the text and idempotent, so this is a no-op
+        # once the store has converged.
+        #
+        # Once per cycle, not once per profile: it reads the whole table in one
+        # sweep with no profile predicate, so running it per profile did the
+        # identical global work N times over and took the write lock N times to
+        # do it. It also takes and releases that lock per batch, so a memory
+        # being saved waits for one batch rather than the whole sweep.
+        #
+        # An earlier version reached for `_conn` and then `.connection`, and the
+        # database manager has neither, so the guard was always False and this
+        # had never run once -- the store drifted further from the rule with
+        # every release while a block that looks like it repairs that sat here
+        # doing nothing.
+        try:
+            from superlocalmemory.storage.migrations import (
+                M048_upcoming_holds_only_what_is_upcoming as _reclassify,
+            )
+            _reclassify.apply(open_connection=self._db.raw_connection)
+            self._record_step("re-reading plans", True)
+        except Exception as exc:  # noqa: BLE001
+            self._record_step("re-reading plans", False, str(exc))
+
+        # Anything that has failed on several cycles running is not a blip.
+        # Every step here logs and continues, which is right -- one broken step
+        # must not stop the rest -- but it meant a step that had been failing
+        # for a week looked exactly like one that had just hiccupped, and the
+        # daemon still reported itself healthy throughout.
+        self._note_step_outcomes()
+
+        # Retention. Three tables had a pruner each, written and wired
+        # separately; the fourth unbounded table was found by reading a
+        # disk-usage report and the fifth by reading the fourth. The policy for
+        # every append-shaped table now lives in one registry and this enforces
+        # all of them, so a table added without a policy is something the test
+        # suite can see rather than something a person has to remember.
+        #
+        # Once per cycle, not per profile: every rule is keyed either on a row's
+        # own age or on whether its referent still exists, and neither is
+        # profile-scoped. Placed after the per-profile work so it sweeps rows
+        # that pass orphaned -- pruning the graph and demoting tiers is what
+        # leaves a lineage or temporal row without a referent.
+        # In pieces, each taking and releasing the write lock. Entering
+        # ``raw_connection`` is what takes that lock, so entering it once for
+        # the whole sweep held it for the whole sweep: measured at 1,480 ms and
+        # 123,888 rows on a 1 GB store, which is most of the budget a save is
+        # allowed, spent waiting.
+        try:
+            from superlocalmemory.storage.retention_policy import (
+                run_retention_bounded,
+            )
+            removed = run_retention_bounded(self._db.raw_connection)
+            if removed:
+                logger.info(
+                    "Retention: %s",
+                    ", ".join(
+                        f"{table} -{count}" for table, count in sorted(removed.items())
+                    ),
+                )
+            self._record_step("retention", True)
+        except Exception as exc:  # noqa: BLE001
+            self._record_step("retention", False, str(exc))
+
         # V3.4.10: Check if auto-backup is due
         try:
             from superlocalmemory.infra.backup import BackupManager