superlocalmemory 4.0.10 → 4.1.0

package/src/superlocalmemory/mcp/tools_core.py

@@ -20,7 +20,7 @@ from typing import Callable
 from mcp.types import ToolAnnotations
 
 from superlocalmemory.core.admission import admits
-from superlocalmemory.core.config import CANONICAL_RECALL_LIMIT
+from superlocalmemory.core.config import CANONICAL_LIST_LIMIT, CANONICAL_RECALL_LIMIT
 from superlocalmemory.core.operation_request import OperationKind
 from superlocalmemory.infra.data_root import state_path
 from superlocalmemory.mcp._daemon_proxy import daemon_unavailable_error
@@ -29,6 +29,19 @@ from superlocalmemory.mcp.shared import authorize_mcp_mutation
 logger = logging.getLogger(__name__)
 
 
+def _projection_queue_depth(db: object) -> int:
+    """Facts queued for the graph and vector projections, or 0 when there are none.
+
+    Imported inside the function: this module is loaded on every MCP start over
+    stdio, where import cost is startup latency a user feels.
+    """
+    try:
+        from superlocalmemory.storage import projection_outbox
+        return projection_outbox.depth(db)
+    except Exception:
+        return 0
+
+
 async def _runtime_profile(get_engine: Callable, explicit: str = "") -> str:
     """Resolve an MCP default profile from daemon runtime truth."""
     if explicit:
@@ -425,6 +438,9 @@ def register_core_tools(server, get_engine: Callable) -> None:
                     "score_contract_version": result.get("score_contract_version", "2"),
                     "calibration_status": result.get("calibration_status", "uncalibrated"),
                     "calibration_id": result.get("calibration_id"),
+                    # Quote this back to report_outcome and the report ties to
+                    # this exact answer instead of being matched by guesswork.
+                    "query_id": result.get("query_id", ""),
                     "answer_confidence": result.get("answer_confidence"),
                     "abstained": result.get("abstained", False),
                     "abstention_reason": result.get("abstention_reason"),
@@ -436,7 +452,7 @@ def register_core_tools(server, get_engine: Callable) -> None:
 
     @server.tool(annotations=ToolAnnotations(readOnlyHint=True))
     @admits(OperationKind.RECALL)
-    async def search(query: str, limit: int = 10) -> dict:
+    async def search(query: str, limit: int = CANONICAL_RECALL_LIMIT) -> dict:
         """Full-text search across memories using FTS5 with BM25 ranking."""
         try:
             engine = get_engine()
@@ -457,6 +473,7 @@ def register_core_tools(server, get_engine: Callable) -> None:
             return {"success": False, "error": str(exc)}
 
     @server.tool(annotations=ToolAnnotations(readOnlyHint=True))
+    @admits(OperationKind.RECALL)
     async def fetch(fact_ids: str) -> dict:
         """Fetch full details for specific fact IDs (comma-separated)."""
         try:
@@ -484,7 +501,8 @@ def register_core_tools(server, get_engine: Callable) -> None:
             return {"success": False, "error": str(exc)}
 
     @server.tool(annotations=ToolAnnotations(readOnlyHint=True))
-    async def list_recent(limit: int = 20) -> dict:
+    @admits(OperationKind.RECALL)
+    async def list_recent(limit: int = CANONICAL_LIST_LIMIT) -> dict:
         """List most recently stored memories, newest first."""
         try:
             engine = get_engine()
@@ -511,6 +529,10 @@ def register_core_tools(server, get_engine: Callable) -> None:
     async def get_status() -> dict:
         """Get memory system status: fact count, entity count, mode, profile, db size."""
         try:
+            # Same source the HTTP surface reads, imported here rather than at
+            # module scope: that module costs ~260ms and MCP starts over stdio.
+            from superlocalmemory.server.routes.helpers import SLM_VERSION
+
             import asyncio
             import os
 
@@ -540,6 +562,10 @@ def register_core_tools(server, get_engine: Callable) -> None:
                         "profile_generation": int(
                             daemon_status.get("profile_generation", 0)
                         ),
+                        "version": SLM_VERSION,
+                        "projection_queue_depth": int(
+                            daemon_status.get("projection_queue_depth", 0)
+                        ),
                     }
 
             engine = get_engine()
@@ -576,6 +602,8 @@ def register_core_tools(server, get_engine: Callable) -> None:
                 "entity_count": entity_count,
                 "edge_count": edge_count,
                 "profile_generation": 0,
+                "version": SLM_VERSION,
+                "projection_queue_depth": _projection_queue_depth(engine._db),
             }
         except Exception as exc:
             logger.exception("get_status failed")

package/src/superlocalmemory/mcp/tools_v28.py

@@ -36,6 +36,7 @@ def register_v28_tools(server, get_engine: Callable) -> None:
         memory_ids: str,
         outcome: str,
         context: str = "",
+        recall_query_id: str = "",
     ) -> dict:
         """Report outcome of using recalled memories.
 
@@ -46,6 +47,10 @@ def register_v28_tools(server, get_engine: Callable) -> None:
             memory_ids: Comma-separated list of fact/memory IDs.
             outcome: One of 'success', 'failure', 'partial'.
             context: Optional freetext context about the outcome.
+            recall_query_id: The ``query_id`` that came back with the recall
+                this report is about. Passing it ties the report to that exact
+                answer; leaving it out falls back to matching on which memories
+                overlap, within a time window.
         """
         try:
             engine = get_engine()
@@ -54,7 +59,20 @@ def register_v28_tools(server, get_engine: Callable) -> None:
                 "update",
                 mutation_source="mcp-report-outcome",
             )
-            from superlocalmemory.learning.outcomes import OutcomeTracker
+            from superlocalmemory.learning.outcomes import (
+                VALID_OUTCOMES,
+                OutcomeTracker,
+            )
+            if outcome not in VALID_OUTCOMES:
+                # Answered here rather than as a stack trace: the caller is an
+                # assistant that can correct itself if told what is allowed.
+                return {
+                    "success": False,
+                    "error": (
+                        f"outcome must be one of {sorted(VALID_OUTCOMES)}, "
+                        f"not {outcome!r}"
+                    ),
+                }
             tracker = OutcomeTracker(engine._db)
             ids = [mid.strip() for mid in memory_ids.split(",") if mid.strip()]
             ctx = {"note": context} if context else None
@@ -64,6 +82,7 @@ def register_v28_tools(server, get_engine: Callable) -> None:
                 outcome=outcome,
                 profile_id=engine.profile_id,
                 context=ctx,
+                recall_query_id=str(recall_query_id or "").strip(),
             )
 
             # v3.4.7: Bridge outcomes → learning signals for two-way learning.

package/src/superlocalmemory/parameterization/pattern_extractor.py

@@ -66,7 +66,17 @@ _BEHAVIORAL_TYPE_MAP: dict[str, str] = {
     "query_type": "workflow_pattern",
     "time_of_day": "workflow_pattern",
     "refinement": "communication_style",
-    "interest": "tech_preference",
+    # An "interest" is a word that shows up often in this user's memories. It is
+    # NOT a statement about their tooling, and calling it one produced the
+    # single worst thing this subsystem has shipped: a prompt injected on every
+    # turn reading "the user's preferred technology stack includes: test, gate,
+    # practices, compliance, projects, while, their, processing, data".
+    #
+    # Every one of those words is a real and frequent topic for this user. The
+    # values were right; the claim about them was false. Measured on a live
+    # store, `_store_patterns` holds correct tech_preference rows alongside
+    # these — Node.js, Go, Git, pip — so the two kinds were never the same kind.
+    "interest": "topic_interest",
     "archival": "avoidance",
 }
 
@@ -90,6 +100,9 @@ class PatternCategory(str, Enum):
     WORKFLOW_PATTERN = "workflow_pattern"
     PROJECT_CONTEXT = "project_context"
     DECISION_HISTORY = "decision_history"
+    # Topics that come up a lot in this user's memories. Deliberately separate
+    # from TECH_PREFERENCE: a frequent word is not a tooling choice.
+    TOPIC_INTEREST = "topic_interest"
     AVOIDANCE = "avoidance"
     CUSTOM = "custom"
 

package/src/superlocalmemory/parameterization/soft_prompt_generator.py

@@ -53,6 +53,11 @@ CATEGORY_TEMPLATES: dict[str, str] = {
         "Current active project: {project_name}. "
         "Key context: {context_summary}."
     ),
+    # Says only what is actually known — that these subjects recur — rather
+    # than inferring a preference, a project or a tool choice from them.
+    "topic_interest": (
+        "Subjects that come up often in the user's notes: {topics}."
+    ),
     "decision_history": (
         "Recent key decisions: {decisions}. "
         "These reflect the user's current direction."
@@ -75,6 +80,7 @@ CATEGORY_PRIORITY_ORDER: list[str] = [
     "behavioral",  # v3.4.7: behavioral assertions after communication style
     "workflow_pattern",
     "project_context",
+    "topic_interest",
     "decision_history",
     "avoidance",
 ]
@@ -105,6 +111,82 @@ class SoftPromptTemplate:
 # SoftPromptGenerator class
 # ---------------------------------------------------------------------------
 
+def _empty_words() -> frozenset[str]:
+    """Words that say nothing about a person when listed as a preference.
+
+    Read from the list the miner already filters on, so there is one definition
+    to extend rather than two that drift.
+    """
+    try:
+        from superlocalmemory.learning.pattern_miner_constants import STOPWORDS
+
+        return frozenset(STOPWORDS)
+    except Exception:  # pragma: no cover — the check degrades to "keep it"
+        return frozenset()
+
+
+_EMPTY_WORDS = _empty_words()
+
+
+def _is_substantive(category: str, values: dict[str, str]) -> bool:
+    """Whether a rendered prompt says anything at all about the user.
+
+    Deliberately weak, and it got that way by being wrong in the other
+    direction first. The original version required a ``tech_preference`` claim to
+    name something from a fixed vocabulary of technologies — which discarded
+    ``Node.js``, ``Git``, ``pip``, ``npm``, ``zig`` and every stack the list did
+    not happen to enumerate. Those were the GENUINE rows on a live store. A
+    filter that silently drops real preferences to catch fake ones is a worse
+    failure than the one it was added for, because nothing reports it.
+
+    What it was actually added for no longer arrives here. The live nonsense —
+    "preferred technology stack includes: test, gate, practices, compliance,
+    projects, while, their, processing, data" — came from word-frequency topics
+    being mapped onto this category, and they are now their own category, where
+    the same words form a true statement. This is the remaining floor: a prompt
+    built entirely out of words that appear in most English sentences says
+    nothing, whatever category it lands in.
+
+    Length is checked per TERM and only against that word list, never as a
+    minimum: "CTO", "AWS", "npm" and "R" are all shorter than a threshold would
+    allow and all mean something.
+    """
+    filled = [
+        v.strip() for v in values.values()
+        if isinstance(v, str) and v.strip()
+    ]
+    if not filled:
+        return False
+    terms = [
+        t.strip().lower()
+        for value in filled
+        for t in value.replace(";", ",").split(",")
+        if t.strip()
+    ]
+    if not terms:
+        return False
+    return any(term not in _EMPTY_WORDS for term in terms)
+
+
+def _fix_stutter(content: str) -> str:
+    """Remove the duplicated conjunction where a template meets its value.
+
+    ``"The user typically {workflow_description}"`` was rendering as "The user
+    typically When when using X" — the template supplies the lead-in and the
+    value already starts with its own. Observed on 19 of 34 stored prompts.
+    """
+    import re as _re
+
+    for word in ("when", "typically", "prefers", "often", "usually"):
+        content = _re.sub(
+            rf"\b({word})\s+{word}\b", r"\1", content, flags=_re.IGNORECASE,
+        )
+    # "typically When when" collapses to "typically When"; then the lead-in and
+    # the value's own opener are adjacent duplicates of different case.
+    content = _re.sub(r"\btypically\s+When\b", "typically, when", content)
+    return content
+
+
 class SoftPromptGenerator:
     """Convert extracted pattern assertions into natural language soft prompts.
 
@@ -243,12 +325,25 @@ class SoftPromptGenerator:
 
         # Clean up
         content = self._clean_content(content)
+        content = _fix_stutter(content)
 
         # Filter PII
         content = self._pii_filter.filter_text(content)
         if not content.strip():
             return None
 
+        # A prompt that says nothing must not be injected. These go into the
+        # model's context on every turn, so an empty claim is not neutral — it
+        # spends the budget and asserts something false. Measured on a live
+        # store: 15 of 34 stored prompts read "the user's preferred technology
+        # stack includes: data, processing, their, projects, test", built from
+        # common words that happened to appear near a technology keyword.
+        if not _is_substantive(category, values):
+            logger.debug(
+                "soft prompt for %r dropped: no substantive values", category,
+            )
+            return None
+
         # Trim to 100 tokens per category
         content = self._trim_to_tokens(content, 100)
 
@@ -305,6 +400,9 @@ class SoftPromptGenerator:
         elif category == "workflow_pattern":
             values["workflow_description"] = "; ".join(pat_values)
 
+        elif category == "topic_interest":
+            values["topics"] = ", ".join(pat_values)
+
         elif category == "project_context":
             values["project_name"] = pat_values[0] if pat_values else ""
             values["context_summary"] = ", ".join(pat_values[1:])

package/src/superlocalmemory/retrieval/bm25_channel.py

@@ -54,6 +54,65 @@ def tokenize(text: str) -> list[str]:
     return [t for t in tokens if t not in _STOPWORDS]
 
 
+#: Saturation constant for the BM25 -> [0,1) transform. Chosen from measurement,
+#: not taste: raw scores on a live store run ~2.5 to ~15 across query
+#: lengths, so k = 5 puts an ordinary match near 0.4 and a strong one near 0.7,
+#: leaving headroom at both ends rather than pinning everything to one corner.
+_BM25_SATURATION = 5.0
+
+
+def _to_unit_scale(scored: list[tuple[str, float]]) -> list[tuple[str, float]]:
+    """Map BM25 scores into [0, 1) without disturbing order or magnitude.
+
+    WHY ANY OF THIS. ``engine.apply_channel_weights`` re-scores a candidate as
+    ``sum(channel_scores[ch] * weights[ch])`` — a SUM of raw channel scores.
+    Every other channel is bounded: semantic cosine and Fisher-Rao are [0, 1],
+    the temporal proximity score is Gaussian on [0, 1]. BM25 is not. Measured on
+    a live store, one query at a time::
+
+        "slm release"                                    max  2.845
+        "memory"                                         max  3.865
+        "the release ships once both audits are clean"    max 10.150
+
+    So the sum was decided by a scale rather than by evidence, and any weight
+    the bandit converged on was a correction for that scale — wrong the moment
+    the query length changed.
+
+    WHY NOT DIVIDE BY THE BATCH MAXIMUM. That was the first implementation here
+    and it is wrong, which an existing test caught before it shipped:
+    ``test_real_fts5_exact_hit_keeps_bounded_slot`` exists to prove *"a real
+    sub-1.0 FTS5 hit remains visible under semantic pressure"*. Dividing by the
+    batch maximum makes the best result exactly 1.0 **whatever it scored**, so a
+    query with one weak lexical match reports full confidence and outranks a
+    semantic channel it should lose to. Batch-relative scaling manufactures
+    confidence out of an empty batch, and it also makes a fact's score depend on
+    which other facts happened to come back — the same query returning a
+    different number on a different day, which HARD-RULES RULE 6 puts above
+    speed.
+
+    A saturating transform has neither problem: ``s / (s + k)`` is strictly
+    increasing, so order is untouched; it is bounded below 1.0, so nothing is
+    ever certain; and it depends only on the score itself, so it is repeatable.
+    Applied to the measurements above: 2.676 -> 0.35, 2.845 -> 0.36,
+    3.865 -> 0.44, 10.150 -> 0.67.
+
+    FUSION IS UNAFFECTED, verified by reading ``fusion.weighted_rrf`` rather
+    than assuming: it computes ``fused += w / (k + rank)`` and keeps the score
+    only for reporting. Rescaling a value nothing divides by cannot move a
+    fused rank.
+
+    Non-positive scores map to 0.0. FTS5's ``bm25()`` is <= 0 and is negated
+    here, so a value at or below zero means "no lexical evidence", and that is
+    what it should contribute to a sum.
+    """
+    if not scored:
+        return scored
+    return [
+        (fid, (s / (s + _BM25_SATURATION)) if s > 0.0 else 0.0)
+        for fid, s in scored
+    ]
+
+
 class BM25Channel:
     """Persistent BM25Plus index for keyword retrieval.
 
@@ -300,10 +359,10 @@ class BM25Channel:
         # Falls back to rank_bm25 ONLY if the FTS5 table is genuinely
         # unavailable (raises) — e.g. a pre-FTS legacy DB.
         try:
-            return self._fts5_search(
+            return _to_unit_scale(self._fts5_search(
                 query, profile_id, top_k,
                 include_global=include_global, include_shared=include_shared,
-            )
+            ))
         except Exception as exc:  # pragma: no cover — legacy/missing FTS table
             logger.debug(
                 "BM25 FTS5 path unavailable, using rank_bm25 fallback: %s", exc,
@@ -338,7 +397,9 @@ class BM25Channel:
                 scored.append((self._fact_ids[i], bonus))
 
         scored.sort(key=lambda x: (-x[1], x[0]))
-        return scored[:top_k]
+        # Same rescale as the FTS5 path. This fallback applies a 1.5x exact
+        # phrase bonus, so its raw ceiling is higher still.
+        return _to_unit_scale(scored[:top_k])
 
     def update_fact(self, fact_id: str, new_content: str, profile_id: str) -> None:
         """Replace a fact's representation in the live index and persist new tokens.

package/src/superlocalmemory/retrieval/channel_status.py

@@ -0,0 +1,117 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under AGPL-3.0-or-later - see LICENSE file
+
+"""What happened to each retrieval channel on one recall.
+
+WHY THIS EXISTS
+---------------
+A channel that crashes on every query and a channel that correctly found
+nothing produced the same observable result: absence. Both simply had no entry
+in the fused candidate map, and the only trace was a log line on a machine
+nobody is reading.
+
+That matters because the channels are not interchangeable. Lexical search
+finding nothing for a conceptual question is the system working. Lexical search
+raising on every question is an outage that looks, from the outside, like a
+store with nothing in it — the user sees weaker answers and has no way to tell
+which of the two they are getting.
+
+There is a third case the absence hid, and it is the worst of them: a channel
+that never ran. When the query embedding is unavailable, three of the five
+channels are never even dispatched. Nothing in the answer said so.
+
+WHY MORE THAN "ok / empty / error"
+----------------------------------
+Because the remedies differ, and a status whose remedy is ambiguous is a status
+nobody acts on. ``error`` is a bug to fix. ``timeout`` is a capacity or data-size
+problem. ``no_embedding`` means the embedding provider is down and several
+channels are silently offline together. ``disabled`` and ``not_configured`` are
+someone's deliberate choice and must not read as faults — which is the point:
+without naming them, an operator reading a list of missing channels cannot tell
+their own configuration apart from a failure.
+
+STRINGS, NOT AN ENUM
+--------------------
+This crosses the MCP and HTTP surfaces, where it is JSON either way. A str-valued
+constant serialises without a custom encoder and compares equal to what a client
+sends back.
+"""
+
+from __future__ import annotations
+
+from typing import Literal
+
+__all__ = [
+    "ALL_STATUSES",
+    "CHANNEL_NAMES",
+    "ChannelStatus",
+    "DISABLED",
+    "EMPTY",
+    "ERROR",
+    "NOT_CONFIGURED",
+    "NO_CANDIDATES",
+    "NO_EMBEDDING",
+    "OK",
+    "TIMEOUT",
+    "is_fault",
+]
+
+ChannelStatus = Literal[
+    "ok",
+    "empty",
+    "error",
+    "timeout",
+    "disabled",
+    "not_configured",
+    "no_embedding",
+    "no_candidates",
+]
+
+#: Ran and contributed candidates.
+OK: ChannelStatus = "ok"
+#: Ran, contributed nothing. A legitimate answer, not a fault.
+EMPTY: ChannelStatus = "empty"
+#: Raised. This answer is missing whatever this channel alone could see.
+ERROR: ChannelStatus = "error"
+#: Abandoned at the hang guard. The answer is incomplete, not merely late.
+TIMEOUT: ChannelStatus = "timeout"
+#: Excluded by configuration or by a per-recall ablation flag.
+DISABLED: ChannelStatus = "disabled"
+#: No such channel on this engine — not built, or its dependency is absent.
+NOT_CONFIGURED: ChannelStatus = "not_configured"
+#: Needed the query embedding, which was unavailable. Several channels fail
+#: together this way, and none of them individually did anything wrong.
+NO_EMBEDDING: ChannelStatus = "no_embedding"
+#: Had nothing to work on. Distinct from ``empty``, which claims a search
+#: happened: a channel that re-scores other channels' candidates never searched
+#: at all when there were none, and if the reason there were none is that the
+#: others failed, calling this "found nothing" hides the actual fault.
+NO_CANDIDATES: ChannelStatus = "no_candidates"
+
+ALL_STATUSES: frozenset[str] = frozenset({
+    OK, EMPTY, ERROR, TIMEOUT, DISABLED, NOT_CONFIGURED, NO_EMBEDDING,
+    NO_CANDIDATES,
+})
+
+#: Every channel a recall can report on, so a caller can tell "this channel had
+#: no status recorded" (a gap in the reporting) from "this channel reported that
+#: it did nothing" (an answer). A missing key is a bug; ``empty`` is not.
+CHANNEL_NAMES: tuple[str, ...] = (
+    "semantic",
+    "bm25",
+    "temporal",
+    "hopfield",
+    "spreading_activation",
+    "entity_graph",
+    "profile",
+)
+
+#: Statuses that mean the answer is worse than it should have been. ``empty``,
+#: ``disabled`` and ``not_configured`` are deliberately absent: the first is a
+#: valid finding and the other two are somebody's decision.
+_FAULTS: frozenset[str] = frozenset({ERROR, TIMEOUT, NO_EMBEDDING})
+
+
+def is_fault(status: str | None) -> bool:
+    """Whether this status means the answer was degraded."""
+    return (status or "") in _FAULTS