superlocalmemory 4.0.9 → 4.1.0

package/src/superlocalmemory/retrieval/channel_status.py

@@ -0,0 +1,117 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under AGPL-3.0-or-later - see LICENSE file
+
+"""What happened to each retrieval channel on one recall.
+
+WHY THIS EXISTS
+---------------
+A channel that crashes on every query and a channel that correctly found
+nothing produced the same observable result: absence. Both simply had no entry
+in the fused candidate map, and the only trace was a log line on a machine
+nobody is reading.
+
+That matters because the channels are not interchangeable. Lexical search
+finding nothing for a conceptual question is the system working. Lexical search
+raising on every question is an outage that looks, from the outside, like a
+store with nothing in it — the user sees weaker answers and has no way to tell
+which of the two they are getting.
+
+There is a third case the absence hid, and it is the worst of them: a channel
+that never ran. When the query embedding is unavailable, three of the five
+channels are never even dispatched. Nothing in the answer said so.
+
+WHY MORE THAN "ok / empty / error"
+----------------------------------
+Because the remedies differ, and a status whose remedy is ambiguous is a status
+nobody acts on. ``error`` is a bug to fix. ``timeout`` is a capacity or data-size
+problem. ``no_embedding`` means the embedding provider is down and several
+channels are silently offline together. ``disabled`` and ``not_configured`` are
+someone's deliberate choice and must not read as faults — which is the point:
+without naming them, an operator reading a list of missing channels cannot tell
+their own configuration apart from a failure.
+
+STRINGS, NOT AN ENUM
+--------------------
+This crosses the MCP and HTTP surfaces, where it is JSON either way. A str-valued
+constant serialises without a custom encoder and compares equal to what a client
+sends back.
+"""
+
+from __future__ import annotations
+
+from typing import Literal
+
+__all__ = [
+    "ALL_STATUSES",
+    "CHANNEL_NAMES",
+    "ChannelStatus",
+    "DISABLED",
+    "EMPTY",
+    "ERROR",
+    "NOT_CONFIGURED",
+    "NO_CANDIDATES",
+    "NO_EMBEDDING",
+    "OK",
+    "TIMEOUT",
+    "is_fault",
+]
+
+ChannelStatus = Literal[
+    "ok",
+    "empty",
+    "error",
+    "timeout",
+    "disabled",
+    "not_configured",
+    "no_embedding",
+    "no_candidates",
+]
+
+#: Ran and contributed candidates.
+OK: ChannelStatus = "ok"
+#: Ran, contributed nothing. A legitimate answer, not a fault.
+EMPTY: ChannelStatus = "empty"
+#: Raised. This answer is missing whatever this channel alone could see.
+ERROR: ChannelStatus = "error"
+#: Abandoned at the hang guard. The answer is incomplete, not merely late.
+TIMEOUT: ChannelStatus = "timeout"
+#: Excluded by configuration or by a per-recall ablation flag.
+DISABLED: ChannelStatus = "disabled"
+#: No such channel on this engine — not built, or its dependency is absent.
+NOT_CONFIGURED: ChannelStatus = "not_configured"
+#: Needed the query embedding, which was unavailable. Several channels fail
+#: together this way, and none of them individually did anything wrong.
+NO_EMBEDDING: ChannelStatus = "no_embedding"
+#: Had nothing to work on. Distinct from ``empty``, which claims a search
+#: happened: a channel that re-scores other channels' candidates never searched
+#: at all when there were none, and if the reason there were none is that the
+#: others failed, calling this "found nothing" hides the actual fault.
+NO_CANDIDATES: ChannelStatus = "no_candidates"
+
+ALL_STATUSES: frozenset[str] = frozenset({
+    OK, EMPTY, ERROR, TIMEOUT, DISABLED, NOT_CONFIGURED, NO_EMBEDDING,
+    NO_CANDIDATES,
+})
+
+#: Every channel a recall can report on, so a caller can tell "this channel had
+#: no status recorded" (a gap in the reporting) from "this channel reported that
+#: it did nothing" (an answer). A missing key is a bug; ``empty`` is not.
+CHANNEL_NAMES: tuple[str, ...] = (
+    "semantic",
+    "bm25",
+    "temporal",
+    "hopfield",
+    "spreading_activation",
+    "entity_graph",
+    "profile",
+)
+
+#: Statuses that mean the answer is worse than it should have been. ``empty``,
+#: ``disabled`` and ``not_configured`` are deliberately absent: the first is a
+#: valid finding and the other two are somebody's decision.
+_FAULTS: frozenset[str] = frozenset({ERROR, TIMEOUT, NO_EMBEDDING})
+
+
+def is_fault(status: str | None) -> bool:
+    """Whether this status means the answer was degraded."""
+    return (status or "") in _FAULTS

package/src/superlocalmemory/retrieval/engine.py

@@ -26,7 +26,12 @@ import threading
 import time
 from typing import TYPE_CHECKING, Any, Protocol
 
-from superlocalmemory.core.config import ChannelWeights, RetrievalConfig
+from superlocalmemory.core.config import (
+    CANONICAL_RECALL_LIMIT,
+    ChannelWeights,
+    RetrievalConfig,
+)
+from superlocalmemory.retrieval import channel_status as chstat
 from superlocalmemory.retrieval.fusion import FusionResult, weighted_rrf
 from superlocalmemory.retrieval.strategy import QueryStrategy, QueryStrategyClassifier
 from superlocalmemory.retrieval.temporal_validity_filter import (
@@ -195,7 +200,7 @@ class RetrievalEngine:
 
     def recall(
         self, query: str, profile_id: str,
-        mode: Mode = Mode.A, limit: int = 20,
+        mode: Mode = Mode.A, limit: int = CANONICAL_RECALL_LIMIT,
         *,
         extra_disabled_channels: set[str] | None = None,
         include_global: bool = False,
@@ -246,19 +251,41 @@ class RetrievalEngine:
         strat = self._strategy.classify(query, self._base_weights)
         _em("classify")
 
-        # Profile shortcut (runs before channel search)
-        if self._profile_channel is not None:
+        # What each channel did, so the caller can tell an answer apart from
+        # an outage. Owned by this call for the same reason the dropped set is:
+        # a dict on the engine would have two concurrent recalls overwriting
+        # each other's report.
+        channel_status: dict[str, str] = {}
+
+        # Profile shortcut (runs before channel search).
+        #
+        # The ablation flag is honoured here too. It was not, so an operator who
+        # switched this channel off still had it searched, still had its weight
+        # doubled on a hit, and read a status of "ok" or "empty" — which reports
+        # their own configuration as a live channel's finding.
+        _profile_disabled = "profile" in set(self._config.disabled_channels) | set(
+            extra_disabled_channels or (),
+        )
+        if _profile_disabled:
+            profile_hits = []
+            channel_status["profile"] = chstat.DISABLED
+        elif self._profile_channel is not None:
             try:
                 profile_hits = self._profile_channel.search(
                     query, profile_id, top_k=10,
                 )
+                channel_status["profile"] = (
+                    chstat.OK if profile_hits else chstat.EMPTY
+                )
                 if profile_hits:
                     strat.weights["profile"] = 2.0
             except Exception as exc:
                 logger.warning("Profile channel: %s", exc)
                 profile_hits = []
+                channel_status["profile"] = chstat.ERROR
         else:
             profile_hits = []
+            channel_status["profile"] = chstat.NOT_CONFIGURED
 
         # Dynamic top-k for aggregation queries
         effective_limit = 100 if strat.query_type == "aggregation" else limit
@@ -276,6 +303,7 @@ class RetrievalEngine:
             as_of=as_of, known_as_of=known_as_of, valid_at=valid_at,
             include_unknown=include_unknown,
             dropped_channels=dropped_channels,
+            channel_status=channel_status,
         )
         _em("run_channels")
         # One request may need admission before fusion and again after optional
@@ -393,9 +421,29 @@ class RetrievalEngine:
         # this stage re-scores every fused candidate and then re-sorts them, so
         # whether it ran decided the top answer outright rather than adding to
         # it. Bound by data if it ever needs bounding, never by elapsed time.
-        if (self._entity is not None
-                and "entity_graph" not in set(self._config.disabled_channels)
-                and fused):
+        if self._entity is None:
+            channel_status["entity_graph"] = chstat.NOT_CONFIGURED
+        elif "entity_graph" in set(self._config.disabled_channels) | set(
+            extra_disabled_channels or (),
+        ):
+            # Both the configured set AND the per-call one, the same way the
+            # profile channel resolves it. Reading only the configured set meant
+            # a caller that switched this channel off for one recall still had
+            # it run, and still saw it reported as though it were the caller's
+            # own setting that was being honoured.
+            channel_status["entity_graph"] = chstat.DISABLED
+        elif not fused:
+            # It did not run, and saying "found nothing" would be a different
+            # claim. This channel re-scores other channels' candidates rather
+            # than producing its own, so with nothing fused there is nothing for
+            # it to do — and if the reason nothing fused is that the other five
+            # failed, reporting this one as having searched and come back empty
+            # hides that.
+            channel_status["entity_graph"] = chstat.NO_CANDIDATES
+        else:
+            # One chain, evaluated once. Repeating the three conditions to guard
+            # the work separately is how a status starts describing a decision
+            # the code no longer makes.
             try:
                 candidate_ids = [fr.fact_id for fr in fused[:100]]
                 eg_scores = self._entity.score_candidates(
@@ -405,6 +453,9 @@ class RetrievalEngine:
                     include_global=include_global,
                     include_shared=include_shared,
                 )
+                channel_status["entity_graph"] = (
+                    chstat.OK if eg_scores else chstat.EMPTY
+                )
                 if eg_scores:
                     boosted = []
                     for fr in fused:
@@ -423,6 +474,7 @@ class RetrievalEngine:
                     fused = sorted(boosted, key=lambda r: (-r.fused_score, r.fact_id))
             except Exception as exc:
                 logger.warning("Entity graph signal enhancement: %s", exc)
+                channel_status["entity_graph"] = chstat.ERROR
 
         # Brain Core S402: bridge and scene expansion append candidates after
         # the channel boundary. Reapply the same hard correction-admission rule
@@ -565,6 +617,7 @@ class RetrievalEngine:
             # community. Precomputed summary lookup only — no per-query LLM.
             community_context=self._community_context(results, profile_id),
             incomplete_channels=tuple(sorted(dropped_channels)),
+            channel_status=dict(channel_status),
         )
 
     # -- Community context (Wave Q2b) --------------------------------------
@@ -977,6 +1030,7 @@ class RetrievalEngine:
         valid_at: str | None = None,
         include_unknown: bool = False,
         dropped_channels: set[str] | None = None,
+        channel_status: dict[str, str] | None = None,
     ) -> dict[str, list[tuple[str, float]]]:
         """Run active retrieval channels.
 
@@ -994,6 +1048,12 @@ class RetrievalEngine:
         than merely late. It is a caller-owned set passed down per recall and
         deliberately not an attribute of self — two concurrent recalls sharing
         one would report each other's losses (the v3.4.64 race).
+
+        ``channel_status``, likewise caller-owned, receives one entry per
+        channel saying what became of it. Every channel gets exactly one:
+        those that cannot run are recorded before dispatch with the reason, and
+        every dispatched channel is recorded by the collection loop below —
+        which iterates the futures, so it cannot skip one.
         """
         import os as _os_e
         import time as _time_e
@@ -1023,23 +1083,50 @@ class RetrievalEngine:
             except Exception as exc:
                 logger.warning("Query embedding failed: %s", exc)
 
+        # Why a channel will not run, recorded BEFORE dispatch. An embedding
+        # failure silently takes three of the five channels down together, and
+        # the answer never said so: it looked exactly like a store with nothing
+        # relevant in it. Configuration and ablation are recorded too, so an
+        # operator reading a list of absent channels can tell their own choices
+        # apart from a fault.
+        if channel_status is not None:
+            for _name, _obj, _needs_emb in (
+                ("semantic", self._semantic, True),
+                ("bm25", self._bm25, False),
+                ("temporal", self._temporal, False),
+                ("hopfield", self._hopfield, True),
+                ("spreading_activation", self._spreading_activation, True),
+            ):
+                if _obj is None:
+                    channel_status[_name] = chstat.NOT_CONFIGURED
+                elif _name in disabled:
+                    channel_status[_name] = chstat.DISABLED
+                elif _needs_emb and q_emb is None:
+                    channel_status[_name] = chstat.NO_EMBEDDING
+
         # v3.4.53: collect channel callables and run in parallel.
         # Each channel is a standalone search — no shared mutable state,
         # no ordering dependencies. SQLite WAL mode permits concurrent reads.
         futures: dict[str, concurrent.futures.Future] = {}
 
         def _safe_channel(name: str, fn, *args):
-            """Run a single channel, returning (name, result_or_None)."""
+            """Run a single channel, returning (name, result_or_None, status).
+
+            Returning the status alongside the result is what separates "found
+            nothing" from "raised": both used to come back as ``None``.
+            """
             _cs = _time_e.monotonic() if _et else 0.0
             try:
                 res = fn(*args)
                 if _et:
                     logger.warning("[RECALL-TIMING]     channel.%-16s %.0f ms",
                                    name, (_time_e.monotonic() - _cs) * 1000.0)
-                return (name, res if res else None)
+                if res:
+                    return (name, res, chstat.OK)
+                return (name, None, chstat.EMPTY)
             except Exception as exc:
                 logger.warning("%s channel: %s", name, exc)
-                return (name, None)
+                return (name, None, chstat.ERROR)
 
         executor = self._channel_executor
         if self._semantic is not None and q_emb is not None and "semantic" not in disabled:
@@ -1111,14 +1198,22 @@ class RetrievalEngine:
                 )
                 if dropped_channels is not None:
                     dropped_channels.add(name)
+                # Same branch as the dropped set on purpose: two writes in one
+                # place cannot disagree about which channels timed out.
+                if channel_status is not None:
+                    channel_status[name] = chstat.TIMEOUT
                 fut.cancel()  # no-op if already running; prevents queued jobs from starting
                 continue
             try:
-                ch_name, result = fut.result()
+                ch_name, result, status = fut.result()
+                if channel_status is not None:
+                    channel_status[ch_name] = status
                 if result:
                     out[ch_name] = result
             except Exception as exc:
                 logger.warning("Channel %s failed: %s", name, exc)
+                if channel_status is not None:
+                    channel_status[name] = chstat.ERROR
 
         # Apply registered post-retrieval filters (forgetting filter, etc.).
         # Pass as_of in context dict when set so the bi-temporal validity filter