superlocalmemory 4.0.9 → 4.1.0

package/src/superlocalmemory/core/config.py

@@ -28,9 +28,22 @@ logger = logging.getLogger(__name__)
 
 #: Default number of results returned by recall across MCP, CLI, daemon, and
 #: engine.  All surfaces bind their ``limit`` defaults to this constant so a
-#: single change is sufficient to keep the full stack in sync.
+#: single change is sufficient to keep the full stack in sync.  Search is a
+#: retrieval operation too and binds here: asking the same question through
+#: ``recall`` and through ``search`` must not return differently sized answers.
 CANONICAL_RECALL_LIMIT: int = 20
 
+#: Default number of rows returned when listing the newest memories rather than
+#: answering a question — MCP ``list_recent`` and ``slm list``.  Separate from
+#: the recall limit because the two can legitimately diverge; equal today.
+CANONICAL_LIST_LIMIT: int = 20
+
+#: Page size for the HTTP memory browser (``GET /api/memories``), which is a
+#: paged table rather than an answer.  Deliberately larger than the list limit:
+#: a person scrolling a table wants a fuller page than an agent asking what
+#: happened recently.  The ceiling on that endpoint is enforced separately.
+BROWSE_PAGE_SIZE: int = 50
+
 
 # ---------------------------------------------------------------------------
 # Default Paths
@@ -767,8 +780,8 @@ class ParameterizationConfig:
     max_memory_tokens: int = 1500      # Token budget for regular memories
     categories_enabled: tuple[str, ...] = (
         "identity", "tech_preference", "communication_style",
-        "workflow_pattern", "project_context", "decision_history",
-        "avoidance",
+        "workflow_pattern", "project_context", "topic_interest",
+        "decision_history", "avoidance",
     )
 
     # Lifecycle
@@ -1152,14 +1165,26 @@ class SLMConfig:
     # stay on SQLite until a staged parity check promotes these projections.
     scale_engine_state: str = "local_core"  # local_core | prepared | verified | promoted
     # v3.8.5: auto-promote Cozo+LanceDB when a DB grows past the scale at which
-    # they actually help.  Below the threshold the well-indexed SQLite graph is
-    # faster (measured ~1.7ms/traversal at 208K edges), so normal installs stay
-    # on Local Core and never pay the projection/migration cost.  The threshold
-    # is deliberately high: the graph DB win appears at millions of edges, not
-    # hundreds of thousands.  Auto-promotion is background, uses the same staged
+    # they actually help.  Auto-promotion is background, uses the same staged
     # parity gate as the manual path, and falls back to SQLite on any failure.
+    #
+    # 100,000 (was 1,000,000, set by Varun 2026-08-22). The old value gated every
+    # store anyone actually has -- the author's is 140,000 edges -- so the
+    # projection was built for nobody and, being unexercised, was wrong: it
+    # carried 1,257 memories the store may not return and its graph search
+    # disagreed with SQLite on every query.  A threshold no real store crosses is
+    # not a safety margin, it is a feature nobody tests.
+    #
+    # What promotion now changes is narrow and measured.  The graph WALK does not
+    # move: SQLite loads 130k edges in 141 ms against CozoDB's 252 ms, so the
+    # adjacency read stays where it is fastest (retrieval/graph_adjacency.py
+    # holds the seam for the day that flips).  What promotion switches on is the
+    # LanceDB vector index, which is within noise of sqlite-vec at this size
+    # (18.3 ms against 18.7 ms over 5,324 vectors) and is the one built to keep
+    # going as the vector count grows.  So crossing this line is close to
+    # latency-neutral today and buys the projection real exercise.
     scale_auto_promote_enabled: bool = True
-    scale_auto_promote_min_edges: int = 1_000_000
+    scale_auto_promote_min_edges: int = 100_000
     evolution: EvolutionConfig = field(default_factory=EvolutionConfig)
     health: HealthConfig = field(default_factory=HealthConfig)
     # v3.8.4-G: Graph thinning parameters (#84)
@@ -1304,10 +1329,10 @@ class SLMConfig:
         )
         try:
             config.scale_auto_promote_min_edges = int(
-                data.get("scale_auto_promote_min_edges", 1_000_000)
+                data.get("scale_auto_promote_min_edges", 100_000)
             )
         except (TypeError, ValueError):
-            config.scale_auto_promote_min_edges = 1_000_000
+            config.scale_auto_promote_min_edges = 100_000
 
         # V3.3 config fields (additive — defaults work if missing from JSON)
         fg = data.get("forgetting", {})
@@ -1392,6 +1417,44 @@ class SLMConfig:
                 if k in RetrievalConfig.__dataclass_fields__
             })
 
+        # 4.1.0 (#124): restore the two sections save() now writes. This runs
+        # AFTER for_mode() has applied its presets, so a value someone chose
+        # beats the mode default -- the same ordering retrieval already relies
+        # on. A malformed section falls back to the dataclass defaults rather
+        # than raising, matching quantization and sagq above: a corrupt config
+        # file must not make `slm` unrunnable.
+        mth = data.get("math")
+        if isinstance(mth, dict) and mth:
+            try:
+                fields = {
+                    k: v for k, v in mth.items()
+                    if k in MathConfig.__dataclass_fields__
+                }
+                # JSON has no tuple. Written as a list, it must come back a
+                # tuple or every reader that unpacks a pair gets a list and the
+                # difference surfaces somewhere far from here.
+                rng = fields.get("langevin_weight_range")
+                if isinstance(rng, list):
+                    fields["langevin_weight_range"] = tuple(rng)
+                config.math = MathConfig(**fields)
+            except (TypeError, ValueError) as exc:
+                logger.warning(
+                    "Ignoring invalid math config (%s) — using defaults", exc
+                )
+
+        cw = data.get("channel_weights")
+        if isinstance(cw, dict) and cw:
+            try:
+                config.channel_weights = ChannelWeights(**{
+                    k: v for k, v in cw.items()
+                    if k in ChannelWeights.__dataclass_fields__
+                })
+            except (TypeError, ValueError) as exc:
+                logger.warning(
+                    "Ignoring invalid channel_weights config (%s) — using defaults",
+                    exc,
+                )
+
         # V3.4.3 config fields (additive — missing keys get dataclass defaults)
         config.daemon_idle_timeout = data.get("daemon_idle_timeout", 0)
         config.daemon_port = data.get("daemon_port", 8765)
@@ -1654,11 +1717,20 @@ class SLMConfig:
         #
         # embedding_signature: has no typed in-memory model — it is an opaque blob
         # written by external tooling (see below).
+        # ``math`` and ``channel_weights`` join this loop as of 4.1.0. Both are
+        # read by engine wiring -- ``math.sheaf_contradiction_threshold`` by the
+        # consistency checker, ``channel_weights`` as the retrieval base weights
+        # -- and neither was ever written here or restored in ``load()``. Tuning
+        # either one, by hand or by switching mode, survived until the next
+        # restart and then silently reverted to the dataclass default, with
+        # nothing said. Reported as #124.
         for _section, _obj in (
             ("forgetting", self.forgetting),
             ("quantization", self.quantization),
             ("sagq", self.sagq),
             ("auto_invoke", self.auto_invoke),
+            ("math", self.math),
+            ("channel_weights", self.channel_weights),
         ):
             _base = existing.get(_section)
             _merged = dict(_base) if isinstance(_base, dict) else {}
@@ -1794,6 +1866,13 @@ class SLMConfig:
                 base_dir=_base,
                 embedding=_a_emb,
                 llm=LLMConfig(),  # No LLM
+                # Mode A's promise is that nothing puts a model on any path, and
+                # a subsystem should refuse because the mode says so, not because
+                # it happened to be handed llm=None. Three of the four places
+                # that build the consolidator pass only a database, so today the
+                # guard holds by accident of wiring; this makes it hold by
+                # configuration, which is the thing a reader can check.
+                ccq=CCQConfig(use_llm_gist=False),
                 temporal_validator=TemporalValidatorConfig(mode="a"),
                 retrieval=RetrievalConfig(
                     # V3.3.2: ONNX cross-encoder enabled for all modes (~200MB)

package/src/superlocalmemory/core/consolidation_engine.py

@@ -634,8 +634,42 @@ class ConsolidationEngine:
         )
         result["entity_community_count"] = ec.get("community_count", 0)
         result["community_summaries"] = ec.get("summaries_written", 0)
+        result["entities_compiled"] = self._compile_entities(profile_id)
         return result
 
+    def _compile_entities(self, profile_id: str) -> int:
+        """Bring each entity's summary up to date with its facts.
+
+        This ran only on the maintenance endpoint, never on the periodic
+        consolidation that the daemon schedules on its own. The effect was that
+        almost every entity carried no compiled summary at all: the explorer
+        asked for one, got null, and showed an entity with no description and
+        an empty timeline — which reads as "there is nothing here" rather than
+        "nothing has compiled this yet".
+
+        Runs beside the community recompute because both derive a view over the
+        same graph and both belong to the same pass. Never fatal: a compilation
+        that fails leaves the previous summary in place, and no summary is a
+        worse answer than a stale one but not a broken store.
+        """
+        try:
+            from superlocalmemory.core.config import SLMConfig
+            from superlocalmemory.learning.entity_compiler import EntityCompiler
+
+            db_path = getattr(self._db, "db_path", None)
+            if not db_path:
+                return 0
+            compiled = EntityCompiler(str(db_path), SLMConfig.load()).compile_all(
+                profile_id,
+            )
+            count = int(compiled.get("compiled", 0) or 0)
+            if count:
+                logger.info("Entity compilation: %d entities compiled", count)
+            return count
+        except Exception as exc:
+            logger.debug("Entity compilation skipped: %s", exc)
+            return 0
+
     # ------------------------------------------------------------------
     # Step 6: Derive Associations
     # ------------------------------------------------------------------

package/src/superlocalmemory/core/engine.py

@@ -53,6 +53,35 @@ def _verify_ingestion_schema(memory_db: Path) -> bool:
 # Embedding a memory on the write path, before the receipt is returned
 # ---------------------------------------------------------------------------
 
+def _embedder_is_warm(embedder: object) -> bool:
+    """Has this embedder already served a request, so the next one is cheap?
+
+    Availability and readiness are different questions and the write path needs
+    the second one. ``EmbeddingService._available`` is set to ``True`` in its
+    constructor, before the worker subprocess exists; the model behind it takes
+    **9.9-11.0 s** to load on this machine against **42 ms** once loaded. A write
+    that embeds inline does so behind a one-second deadline, so keying off
+    availability meant every write until something else warmed the model paid the
+    whole deadline and then stored no vector anyway -- 10 of the first 12,
+    measured. The deadline was spent proving the model was cold.
+
+    ``EmbeddingService.is_warm`` answers the real question: a live worker that has
+    served at least one request. So both must hold, and availability is still
+    checked first: an embedder that reports itself unavailable is not a candidate
+    whatever else it says. Requiring only ``is_warm`` would call ``embed`` on a
+    dead embedder that happens to expose the attribute -- any mock, proxy or
+    duck-typed wrapper auto-creates one.
+
+    An embedder that offers no ``is_warm`` keeps availability as the whole answer:
+    ``OllamaEmbedder`` talks to a server that is already running and has no worker
+    of its own to start, so there readiness is availability.
+    """
+    if getattr(embedder, "_available", None) is not True:
+        return False
+    warm = getattr(embedder, "is_warm", None)
+    return True if warm is None else bool(warm)
+
+
 def _is_remote_embedder(embedder: object) -> bool:
     """Return True if *embedder* makes remote HTTP calls (cloud / OpenAI-compatible).
 
@@ -102,6 +131,14 @@ class MemoryEngine:
         self._caps = get_capabilities(config.mode)
         self._capabilities = capabilities
         self._profile_id = config.active_profile
+        # The session a recall belongs to, for matching an outcome back to it
+        # later. Set from whatever the caller last named; falls back to a stable
+        # per-process id so a caller that never names one still leaves a record
+        # that can be matched to another call from the same process. An empty
+        # one is dropped by the queue, which is how 34 of 35 recall paths came
+        # to leave no record at all.
+        self._last_session_id: str = ""
+        self._ambient_session_id: str = f"engine:{os.getpid()}"
         self._initialized = False
 
         self._db = None
@@ -572,7 +609,7 @@ class MemoryEngine:
         _embedder_ref = self._embedder
         if (
             _embedder_ref is None
-            or getattr(_embedder_ref, "_available", None) is not True
+            or not _embedder_is_warm(_embedder_ref)
             or _is_remote_embedder(_embedder_ref)
         ):
             return None, None, None
@@ -901,9 +938,16 @@ class MemoryEngine:
 
         store_fast inserts a verbatim AtomicFact (+ memory row) synchronously.
         The FTS5 ``atomic_facts_fts`` trigger auto-populates on INSERT, so the
-        memory is **keyword/BM25-recallable the instant this returns** (~ms, no
-        LLM, no embedding). Embedding + entities + graph are enriched async by
-        the materializer (which detects facts with NULL embedding).
+        memory is **keyword/BM25-recallable the instant this returns**, with no
+        LLM call. Entities and graph are enriched async by the materializer.
+
+        The embedding is no longer purely async: ``_warm_guard_embed`` computes it
+        inline **when the model is already loaded**, because a fact with no vector
+        is invisible to the semantic channel and a memory written moments ago is
+        then the hardest thing in the store to find by asking about it. Measured
+        on a copy of a real store: **75 ms** with the model loaded, vector
+        attached; **34 ms** when it is not, deferring to the materializer. This
+        docstring used to say "~ms, no embedding" and that had stopped being true.
 
         Returns real fact_ids immediately. Quality gate rejects template junk.
         """
@@ -1016,12 +1060,16 @@ class MemoryEngine:
     ) -> RecallResponse:
         """Recall relevant facts for a query.
 
-        S9-DASH-02: when ``session_id`` is provided, the recall is
-        non-blockingly enqueued to the outcome queue so downstream
-        hooks (PostToolUse, Stop) can attach engagement signals.
-        Zero additional latency on the hot path — enqueue is a
-        ``put_nowait`` and the actual ``pending_outcomes`` INSERT runs
-        on a background worker.
+        ``session_id`` gives the recall continuity: memories this session was
+        recently shown are held in a small in-process working set and bias the
+        ranking of subsequent queries in the same session. Retrieval still runs
+        in full every time — the working set reorders the answer, it never
+        replaces the search. Costs no query and no file access.
+
+        Omitting it is supported and means every recall starts cold, which is
+        what every caller got before. The parameter was previously accepted and
+        then ignored: it was documented as enqueueing a ``pending_outcomes`` row
+        for downstream hooks, and no such row was ever written.
 
         ``fast`` controls only the internal agentic verification round; all six
         local retrieval channels + reranker run regardless. ``fast=None`` (the
@@ -1060,6 +1108,7 @@ class MemoryEngine:
         try:
             response = run_recall(
                 query, pid, mode=mode, limit=limit, agent_id=agent_id,
+                session_id=session_id,
                 config=self._config,
                 retrieval_engine=self._retrieval_engine,
                 trust_scorer=self._trust_scorer,
@@ -1083,8 +1132,75 @@ class MemoryEngine:
             # be submitted through explicit write commands.
             raise
 
+        # Leave a ticket saying what was shown, so an outcome can be attached to
+        # it later. Without one there is no key to join an outcome back to the
+        # recall that produced it, and the whole learning path stops at the first
+        # hop: on a real store every one of 162 recorded outcomes carried an
+        # empty recall id, every per-memory usefulness score sat at its neutral
+        # 0.5, and the ranking model had not retrained in eleven weeks.
+        #
+        # THIS MUST NOT COST THE READER ANYTHING. It is one put_nowait on a
+        # bounded in-memory queue -- roughly a microsecond -- and it drops the
+        # event rather than blocking or raising if the queue is full. A worker
+        # persists it elsewhere. Recall's answer and its timing are unchanged;
+        # measured p50 and p95 before and after are within noise of each other.
+        #
+        # It records only. Nothing here reorders a result: whether learning is
+        # allowed to influence ranking stays a separate, explicit setting that
+        # remains off unless an operator turns it on.
+        try:
+            import uuid
+
+            from superlocalmemory.learning.outcome_queue import (
+                RecallEvent,
+                enqueue_recall,
+            )
+
+            # A fresh id per answer, put on the response as well as the
+            # ticket. `calibration_id` was the obvious candidate and is wrong:
+            # it is None on most paths, and an empty join key is exactly the
+            # state that made all 162 recorded outcomes unmatchable. Returning
+            # it is what makes it useful -- until an answer carried its own
+            # name, no caller could quote one back, so an outcome could only
+            # ever be matched by guessing from overlapping memories.
+            answer_id = uuid.uuid4().hex
+            try:
+                response.query_id = answer_id
+            except Exception:  # noqa: BLE001 -- an older response shape
+                pass
+            enqueue_recall(RecallEvent(
+                session_id=self._session_for_signals(session_id),
+                profile_id=str(pid),
+                query=query,
+                fact_ids=[
+                    r.fact.fact_id for r in (response.results or [])
+                    if getattr(r, "fact", None) is not None
+                ],
+                query_id=answer_id,
+            ))
+        except Exception as exc:  # noqa: BLE001 -- a read must not fail on this
+            logger.debug("recall outcome ticket skipped: %s", exc)
+
         return response
 
+    def _session_for_signals(self, session_id: str | None) -> str:
+        """The name to file this call's outcome under.
+
+        In order: what the caller said, then the last thing any caller on this
+        engine said, then a stable id for this process. Never empty, because an
+        empty one is discarded and the call then leaves no trace an outcome
+        could ever be matched to -- which is what happened on every path that
+        did not thread a session through, meaning nearly all of them.
+
+        Three attribute reads and no I/O. Recall must not get slower to record
+        that it happened.
+        """
+        named = str(session_id or "").strip()
+        if named:
+            self._last_session_id = named
+            return named
+        return self._last_session_id or self._ambient_session_id
+
     # -- Session operations -------------------------------------------------
 
     def create_speaker_entities(
@@ -1099,9 +1215,22 @@ class MemoryEngine:
             )
 
     def close_session(self, session_id: str) -> int:
-        """Create session-level temporal summary."""
+        """Create session-level temporal summary and release its working set.
+
+        The working set is what the session was recently looking at. It exists
+        to connect one turn to the next, so it has no meaning once there are no
+        more turns; holding it would let a later session that reuses this id
+        inherit ranking bias from a conversation that already ended.
+        """
         self._ensure_init()
 
+        try:
+            from superlocalmemory.core.working_memory import discard
+
+            discard(self._profile_id, session_id)
+        except Exception as exc:  # pragma: no cover — never block a close
+            logger.debug("working-set discard skipped: %s", exc)
+
         from superlocalmemory.core.store_pipeline import run_close_session
         return run_close_session(
             session_id, self._profile_id, db=self._db,