superlocalmemory 4.0.10 → 4.1.2

package/src/superlocalmemory/core/graph_pruner.py

@@ -191,17 +191,138 @@ def _batch_delete_by_ids(
     for start in range(0, len(ids), _BATCH_SIZE):
         batch = ids[start:start + _BATCH_SIZE]
         ph = ",".join("?" * len(batch))
+        endpoints = _endpoints_of(db, table, id_col, batch)
         with db.transaction():
             db.execute(
                 f"DELETE FROM {table} WHERE {id_col} IN ({ph})",
                 tuple(batch),
             )
+            # Inside the same transaction as the delete. The graph also lives in
+            # a second store, which no SQLite transaction can reach, so the
+            # durable record of "these facts need re-projecting" has to be as
+            # durable as the delete itself. Without it the second store keeps
+            # serving edges this function just removed, and a search walks a hop
+            # that no longer exists.
+            _enqueue_reprojection(db, endpoints)
         removed += len(batch)
         if start + _BATCH_SIZE < len(ids):
             time.sleep(_BATCH_YIELD_S)
     return removed
 
 
+#: The two columns naming an edge's ends, per table. ``association_edges``
+#: calls them ``source_fact_id``/``target_fact_id``, so a helper that assumed
+#: ``source_id``/``target_id`` raised there, returned nothing, and every
+#: association-edge deletion went unannounced to the graph store -- silently,
+#: because the failure looked exactly like "this table has no endpoints".
+_ENDPOINT_COLUMNS: dict[str, tuple[str, str]] = {
+    "graph_edges": ("source_id", "target_id"),
+    "association_edges": ("source_fact_id", "target_fact_id"),
+}
+
+
+def _endpoints_of(
+    db: "DatabaseManager", table: str, id_col: str, ids: list,
+) -> list[tuple[str, str]]:
+    """``(fact_id, profile_id)`` for both ends of the rows about to be deleted.
+
+    Read before the delete, because afterwards there is nothing to read. Only
+    the ends that are facts matter: an entity id here is projected as part of
+    whichever facts reference it, and those facts are re-derived anyway.
+    """
+    if not ids:
+        return []
+    columns = _ENDPOINT_COLUMNS.get(table)
+    if columns is None:
+        # A table nobody has named the endpoints of is one this pass must not
+        # guess at. Loudly, because guessing wrong is what produced the silent
+        # hole above.
+        logger.warning(
+            "prune: %s has no declared endpoint columns, so its deletions "
+            "cannot be announced to the graph projection", table,
+        )
+        return []
+    ph = ",".join("?" * len(ids))
+    try:
+        rows = db.execute(
+            f"SELECT {columns[0]}, {columns[1]}, profile_id FROM {table} "
+            f"WHERE {id_col} IN ({ph})",
+            tuple(ids),
+        )
+    except Exception as exc:  # noqa: BLE001
+        logger.warning("prune: cannot read endpoints from %s: %s", table, exc)
+        return []
+    seen: dict[tuple[str, str], None] = {}
+    for row in rows:
+        record = dict(row)
+        profile_id = str(record.get("profile_id") or "default")
+        for column in columns:
+            value = record.get(column)
+            if value:
+                seen[(str(value), profile_id)] = None
+    return list(seen)
+
+
+def _enqueue_reprojection(
+    db: "DatabaseManager", endpoints: list[tuple[str, str]],
+) -> None:
+    """Queue an upsert for each fact whose edges just changed.
+
+    The queue coalesces on fact id and the worker re-reads the fact's current
+    edges from SQLite, so queueing an endpoint twice, or queueing one whose
+    edges were already correct, costs one row and converges on the same answer.
+    An id that is an entity rather than a fact is filtered out here rather than
+    left for the worker, which would otherwise spend a lookup discovering the
+    same thing on every cycle.
+    """
+    if not endpoints:
+        return
+    try:
+        from superlocalmemory.storage import projection_outbox
+    except Exception as exc:  # noqa: BLE001
+        logger.debug("prune: no projection queue module: %s", exc)
+        return
+    try:
+        if not projection_outbox.is_available(db):
+            return
+    except Exception as exc:  # noqa: BLE001
+        logger.debug("prune: projection queue unavailable: %s", exc)
+        return
+    by_profile: dict[str, list[str]] = {}
+    for fact_id, profile_id in endpoints:
+        by_profile.setdefault(profile_id, []).append(fact_id)
+    for profile_id, fact_ids in by_profile.items():
+        ph = ",".join("?" * len(fact_ids))
+        try:
+            rows = db.execute(
+                f"SELECT fact_id FROM atomic_facts WHERE profile_id = ? "
+                f"AND fact_id IN ({ph})",
+                (profile_id, *fact_ids),
+            )
+            real = [dict(row)["fact_id"] for row in rows]
+        except Exception as exc:  # noqa: BLE001
+            logger.debug("prune: cannot confirm endpoints are facts: %s", exc)
+            continue
+        if not real:
+            continue
+        # Deliberately allowed to raise. This runs inside the same transaction
+        # as the delete, so a failure here rolls the delete back and the batch
+        # is simply retried on the next pass -- nothing is lost and nothing
+        # diverges.
+        #
+        # Swallowing it, as this once did, committed the delete with no record
+        # that the graph needed telling. The queue would then be empty, which
+        # is exactly what "the graph is up to date" looks like, and the graph
+        # would serve a link the store had removed with nothing anywhere
+        # recording that it happened. That is the failure this queue exists to
+        # make impossible, and it is the module's own stated policy: "a
+        # durability mechanism that silently degrades to best-effort is the
+        # defect it exists to remove."
+        projection_outbox.enqueue_many(
+            db, real, profile_id, op=projection_outbox.OP_UPSERT,
+        )
+
+
 def _remove_orphan_edges_batched(
     db: "DatabaseManager",
     profile_id: str,

package/src/superlocalmemory/core/maintenance_scheduler.py

@@ -67,6 +67,16 @@ class MaintenanceScheduler:
         self._initial_gc_timer = threading.Timer(90.0, self._initial_cache_gc)
         self._initial_gc_timer.daemon = True
         self._initial_gc_timer.start()
+        # An upgrade arrives with whatever backlog the previous version left.
+        # Waiting a full interval for the first graph-metrics pass would mean
+        # half an hour of ranking memories as though they had no position in the
+        # graph, on exactly the store that just gained the fix. Staggered behind
+        # the cache GC so the two never contend for the write lock.
+        self._initial_metrics_timer = threading.Timer(
+            150.0, self._initial_graph_metrics,
+        )
+        self._initial_metrics_timer.daemon = True
+        self._initial_metrics_timer.start()
         logger.info(
             "Maintenance scheduler started (interval=%dm)",
             self._config.forgetting.scheduler_interval_minutes,
@@ -86,6 +96,29 @@ class MaintenanceScheduler:
         except Exception as exc:
             logger.debug("Startup activation-cache GC skipped: %s", exc)
 
+    def _initial_graph_metrics(self) -> None:
+        """One-shot catch-up so an upgrade does not rank on stale metrics."""
+        if not self._running:
+            return
+        try:
+            from superlocalmemory.core.graph_metrics import (
+                compute_graph_metrics,
+                metrics_are_stale,
+            )
+            for profile_id in self._profile_ids():
+                stale, why = metrics_are_stale(self._db, profile_id)
+                if not stale:
+                    continue
+                report = compute_graph_metrics(self._db, profile_id)
+                if report.ok:
+                    logger.info(
+                        "Graph metrics at startup (%s): %s", why, report.summary(),
+                    )
+                else:
+                    logger.warning("Graph metrics at startup: %s", report.summary())
+        except Exception as exc:
+            logger.debug("Startup graph metrics skipped: %s", exc)
+
     def stop(self) -> None:
         """Stop the scheduler. Idempotent."""
         self._running = False
@@ -96,6 +129,10 @@ class MaintenanceScheduler:
         if _gc_timer is not None:
             _gc_timer.cancel()
             self._initial_gc_timer = None
+        _metrics_timer = getattr(self, "_initial_metrics_timer", None)
+        if _metrics_timer is not None:
+            _metrics_timer.cancel()
+            self._initial_metrics_timer = None
         logger.info("Maintenance scheduler stopped")
 
     def _schedule_next(self) -> None:
@@ -106,6 +143,47 @@ class MaintenanceScheduler:
         self._timer.daemon = True
         self._timer.start()
 
+    #: Consecutive failures of one step before it stops being a hiccup.
+    _ESCALATE_AFTER = 3
+
+    def _record_step(self, step: str, ok: bool, detail: str = "") -> None:
+        """Remember whether a maintenance step worked, and say so when it has
+        stopped working."""
+        counts = getattr(self, "_step_failures", None)
+        if counts is None:
+            counts = self._step_failures = {}
+        if ok:
+            if counts.pop(step, 0):
+                logger.info("maintenance: %s is working again", step)
+            return
+        counts[step] = counts.get(step, 0) + 1
+        if counts[step] >= self._ESCALATE_AFTER:
+            logger.error(
+                "maintenance: %s has failed %d cycles in a row (%s). This is "
+                "not a transient failure; the work it does is not being done.",
+                step, counts[step], detail or "no detail",
+            )
+        else:
+            logger.warning("maintenance: %s failed (%s)", step, detail or "")
+
+    def failing_steps(self) -> dict[str, int]:
+        """Steps that have failed on consecutive cycles, and how many.
+
+        Read by the status surfaces, so an operator can see a persistently
+        broken maintenance step instead of having to find it in the log.
+        """
+        return dict(getattr(self, "_step_failures", {}) or {})
+
+    def _note_step_outcomes(self) -> None:
+        """Escalate anything still failing after this cycle's steps."""
+        failing = self.failing_steps()
+        if failing:
+            logger.warning(
+                "maintenance: %d step(s) still failing: %s",
+                len(failing),
+                ", ".join(f"{name} x{count}" for name, count in sorted(failing.items())),
+            )
+
     def _run(self) -> None:
         """Execute maintenance + auto-backup check, then schedule next run."""
         if not self._running:
@@ -183,6 +261,66 @@ class MaintenanceScheduler:
             except Exception as exc:
                 logger.debug("Graph pruning skipped for %s: %s", profile_id, exc)
 
+            # Pruning the graph orphans the lineage of every edge it removed,
+            # and nothing had ever deleted from that table — on a real store it
+            # had grown to 39% rows describing edges that no longer existed.
+            # This runs immediately after so the rows the pass just orphaned are
+            # collected in the same pass.
+            try:
+                from superlocalmemory.storage.lineage_retention import (
+                    prune_orphan_lineage,
+                )
+                report = prune_orphan_lineage(self._db, profile_id=profile_id)
+                if report.total:
+                    logger.info(
+                        "Lineage retention for %s: %d row(s) removed (%s)",
+                        profile_id, report.total, report.deleted,
+                    )
+            except Exception as exc:
+                logger.debug("Lineage retention skipped for %s: %s", profile_id, exc)
+
+            # Structural metrics. Recall multiplies a candidate's activation by
+            # its PageRank at every hop and biases it toward its query seeds'
+            # communities, and both numbers live in fact_importance -- so a
+            # memory missing from that table is found by the walk and then
+            # ranked as though it had no position in the graph.
+            #
+            # Nothing scheduled this. It ran only when a consolidation happened
+            # to fire or someone called the HTTP endpoint by hand, and on the
+            # author's store that meant one run in nine days: 1,036 of 4,034
+            # visible memories had no score and no community, and the newest
+            # four days of memories had none at all. This runs after pruning so
+            # it describes the graph that pruning left behind.
+            try:
+                from superlocalmemory.core.graph_metrics import (
+                    compute_graph_metrics,
+                    metrics_are_stale,
+                )
+                stale, why = metrics_are_stale(self._db, profile_id)
+                if stale:
+                    backend = None
+                    try:
+                        from superlocalmemory.core.backend_orchestrator import (
+                            get_orchestrator,
+                        )
+                        orchestrator = get_orchestrator()
+                        if orchestrator is not None:
+                            backend = orchestrator.get_graph_backend()
+                    except Exception:  # noqa: BLE001 -- in-process is the default anyway
+                        backend = None
+                    report = compute_graph_metrics(
+                        self._db, profile_id, backend=backend,
+                    )
+                    if report.ok:
+                        logger.info("Graph metrics (%s): %s", why, report.summary())
+                    else:
+                        logger.warning("Graph metrics: %s", report.summary())
+                else:
+                    logger.debug("Graph metrics up to date for %s", profile_id)
+                self._record_step("graph metrics", True)
+            except Exception as exc:  # noqa: BLE001
+                self._record_step("graph metrics", False, str(exc))
+
             # Lifecycle evaluation must cover every stored profile, not only
             # whichever profile was active when the engine started.
             try:
@@ -201,6 +339,73 @@ class MaintenanceScheduler:
             except Exception as exc:
                 logger.debug("Core-block recompile skipped for %s: %s", profile_id, exc)
 
+        # Re-read what is filed as a plan. The one-time pass runs as a
+        # migration; the rule it uses keeps getting sharper, and a completed
+        # migration is never replayed — so without this the store drifts
+        # further from the rule with every release and nothing repairs it. The
+        # pass is a pure function of the text and idempotent, so this is a no-op
+        # once the store has converged.
+        #
+        # Once per cycle, not once per profile: it reads the whole table in one
+        # sweep with no profile predicate, so running it per profile did the
+        # identical global work N times over and took the write lock N times to
+        # do it. It also takes and releases that lock per batch, so a memory
+        # being saved waits for one batch rather than the whole sweep.
+        #
+        # An earlier version reached for `_conn` and then `.connection`, and the
+        # database manager has neither, so the guard was always False and this
+        # had never run once -- the store drifted further from the rule with
+        # every release while a block that looks like it repairs that sat here
+        # doing nothing.
+        try:
+            from superlocalmemory.storage.migrations import (
+                M048_upcoming_holds_only_what_is_upcoming as _reclassify,
+            )
+            _reclassify.apply(open_connection=self._db.raw_connection)
+            self._record_step("re-reading plans", True)
+        except Exception as exc:  # noqa: BLE001
+            self._record_step("re-reading plans", False, str(exc))
+
+        # Anything that has failed on several cycles running is not a blip.
+        # Every step here logs and continues, which is right -- one broken step
+        # must not stop the rest -- but it meant a step that had been failing
+        # for a week looked exactly like one that had just hiccupped, and the
+        # daemon still reported itself healthy throughout.
+        self._note_step_outcomes()
+
+        # Retention. Three tables had a pruner each, written and wired
+        # separately; the fourth unbounded table was found by reading a
+        # disk-usage report and the fifth by reading the fourth. The policy for
+        # every append-shaped table now lives in one registry and this enforces
+        # all of them, so a table added without a policy is something the test
+        # suite can see rather than something a person has to remember.
+        #
+        # Once per cycle, not per profile: every rule is keyed either on a row's
+        # own age or on whether its referent still exists, and neither is
+        # profile-scoped. Placed after the per-profile work so it sweeps rows
+        # that pass orphaned -- pruning the graph and demoting tiers is what
+        # leaves a lineage or temporal row without a referent.
+        # In pieces, each taking and releasing the write lock. Entering
+        # ``raw_connection`` is what takes that lock, so entering it once for
+        # the whole sweep held it for the whole sweep: measured at 1,480 ms and
+        # 123,888 rows on a 1 GB store, which is most of the budget a save is
+        # allowed, spent waiting.
+        try:
+            from superlocalmemory.storage.retention_policy import (
+                run_retention_bounded,
+            )
+            removed = run_retention_bounded(self._db.raw_connection)
+            if removed:
+                logger.info(
+                    "Retention: %s",
+                    ", ".join(
+                        f"{table} -{count}" for table, count in sorted(removed.items())
+                    ),
+                )
+            self._record_step("retention", True)
+        except Exception as exc:  # noqa: BLE001
+            self._record_step("retention", False, str(exc))
+
         # V3.4.10: Check if auto-backup is due
         try:
             from superlocalmemory.infra.backup import BackupManager

package/src/superlocalmemory/core/mode_capability.py

@@ -0,0 +1,111 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under AGPL-3.0-or-later - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""What the running mode can do with a model, in words a user can act on.
+
+WHY THIS EXISTS
+
+Mode A runs with no language model at all. That is the point of it: nothing on
+the store or recall path calls out to anything, and a summary is assembled
+directly from the user's own notes rather than written.
+
+The surfaces already reported *what* they did -- a summary came back labelled
+"assembled directly from your own notes" -- but never *why*, and never what to do
+about it. Someone looking at a plainer summary than they expected had no way to
+learn that a written one needs a model and which modes have one. They would
+reasonably conclude the feature was broken.
+
+So this returns the mode, whether a model is available, and one sentence naming
+the next step. Every model-backed surface returns the same block, so the
+explanation is identical wherever it appears rather than reworded per pane.
+
+WHAT IT IS NOT
+
+It is not a capability gate. Nothing here decides whether a call is made; the
+mode's configuration does that. This only describes the decision to the person
+looking at the result.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+#: Named so the message can say which modes to consider without hard-coding the
+#: sentence at each call site.
+_MODE_WITH_LOCAL_MODEL = "B"
+_MODE_WITH_CLOUD_MODEL = "C"
+
+_NO_MODEL_MESSAGE = (
+    "This mode runs entirely on your machine with no language model, so "
+    "summaries are assembled from your own notes rather than written. For "
+    f"written summaries and the other model-backed features, switch to Mode "
+    f"{_MODE_WITH_LOCAL_MODEL} (a local model) or Mode {_MODE_WITH_CLOUD_MODEL} "
+    "(your own cloud model) in Settings and connect one."
+)
+
+_MODEL_CONFIGURED_BUT_ABSENT = (
+    "This mode uses a language model, but none is reachable right now, so "
+    "results fall back to being assembled from your own notes. Check the model "
+    "settings, and that the local model server is running if you are using one."
+)
+
+_CLOUD_MODEL_HAS_NO_KEY = (
+    "This mode uses a hosted language model and no key has been set for it, so "
+    "summaries are assembled from your own notes rather than written. Run "
+    "`slm provider set` and supply your key, or switch to Mode "
+    f"{_MODE_WITH_LOCAL_MODEL} to use a model on this machine."
+)
+
+#: Providers that run on the machine and need no credential. Anything else is
+#: a hosted service, and naming one without a key is not a configured model --
+#: which is what this used to report, so the surfaces said everything was fine
+#: while every model-backed feature was about to fall back.
+_LOCAL_PROVIDERS = frozenset({
+    "ollama", "lmstudio", "llamacpp", "llama.cpp", "local", "vllm", "none", "",
+})
+
+
+def llm_capability(config: Any, *, llm_reachable: bool | None = None) -> dict:
+    """Describe this mode's model support for a user-facing surface.
+
+    ``llm_reachable`` lets a caller that already knows the answer pass it in --
+    a summary route has just tried and knows whether it worked, and asking again
+    would mean a second connection attempt to say something it already knows.
+    Left as None, availability is taken from configuration alone.
+    """
+    mode = ""
+    provider = ""
+    api_key = ""
+    try:
+        mode = str(getattr(getattr(config, "mode", None), "value", "") or "").upper()
+        llm = getattr(config, "llm", None)
+        provider = str(getattr(llm, "provider", "") or "")
+        api_key = str(getattr(llm, "api_key", "") or "").strip()
+    except Exception as exc:  # noqa: BLE001 -- a description must not raise
+        logger.debug("mode capability: cannot read config: %s", exc)
+
+    mode_has_model = mode in (_MODE_WITH_LOCAL_MODEL, _MODE_WITH_CLOUD_MODEL)
+    needs_a_key = provider.lower() not in _LOCAL_PROVIDERS
+    missing_key = mode_has_model and bool(provider) and needs_a_key and not api_key
+    configured = bool(provider) and mode_has_model and not missing_key
+    available = configured if llm_reachable is None else bool(llm_reachable)
+
+    if not mode_has_model:
+        message = _NO_MODEL_MESSAGE
+    elif missing_key and llm_reachable is not True:
+        message = _CLOUD_MODEL_HAS_NO_KEY
+    elif not available:
+        message = _MODEL_CONFIGURED_BUT_ABSENT
+    else:
+        message = ""
+
+    return {
+        "mode": mode or "?",
+        "llm_available": bool(available),
+        "llm_provider": provider,
+        "message": message,
+    }