superlocalmemory 3.7.8 → 3.8.1

package/src/superlocalmemory/retrieval/engine.py

@@ -17,6 +17,7 @@ License: AGPL-3.0-or-later
 from __future__ import annotations
 
 import concurrent.futures
+import functools
 import logging
 import math
 import re
@@ -26,6 +27,11 @@ from typing import TYPE_CHECKING, Any, Protocol
 
 from superlocalmemory.core.config import ChannelWeights, RetrievalConfig
 from superlocalmemory.retrieval.fusion import FusionResult, weighted_rrf
+from superlocalmemory.retrieval.time_window import (
+    in_window,
+    infer_window_from_query,
+    parse_window,
+)
 from superlocalmemory.retrieval.strategy import QueryStrategy, QueryStrategyClassifier
 from superlocalmemory.storage.models import (
     AtomicFact,
@@ -97,12 +103,11 @@ class RetrievalEngine:
         self._profile_channel = profile_channel
         self._bridge = bridge_discovery
         self._trust_scorer = trust_scorer
-        # v3.6.15: serialise the per-recall scope-flag set + channel execution.
-        # Channel instances are SHARED across concurrent recalls (the daemon runs
-        # several in parallel); without this, recall B's flags could overwrite
-        # recall A's mid-flight on the shared channels. Uncontended for a single
-        # recall (~0 cost); only the channel phase of concurrent recalls serialises.
-        self._scope_lock = threading.Lock()
+        # v3.7.9: scope flags (include_global / include_shared) are now threaded
+        # as explicit call parameters into every channel's search() method, so
+        # concurrent recalls each carry their own flags — no shared mutable state,
+        # no lock needed. The _scope_lock and per-recall attribute-set loop have
+        # been removed. See defect S01 in the fix/3.7.9 branch notes.
         # One executor belongs to one retrieval engine. Creating/destroying six
         # worker threads on every recall caused allocator/thread-stack RSS churn
         # under sustained sessions. The scope lock already serializes channel
@@ -149,6 +154,7 @@ class RetrievalEngine:
         extra_disabled_channels: set[str] | None = None,
         include_global: bool = False,
         include_shared: bool = False,
+        window: str | tuple[str, str] | None = None,
     ) -> RecallResponse:
         """Full retrieval pipeline: strategy -> channels -> RRF -> rerank.
 
@@ -161,7 +167,11 @@ class RetrievalEngine:
         for the ``--fast`` CLI flag) without mutating shared config.
         """
         t0 = time.monotonic()
-        self._extra_disabled = set(extra_disabled_channels or ())
+        # NOTE: extra_disabled_channels is passed as an explicit local argument
+        # to _run_channels() — it is NOT stored on self.  Storing it as a shared
+        # mutable instance attribute (the old self._extra_disabled = ...) caused
+        # a race condition where two concurrent recalls could overwrite each
+        # other's channel-disable set (v3.4.64 fix).
 
         # v3.5.0 diagnostic: stage timing inside retrieval (SLM_RECALL_TIMING=1).
         import os as _os_e
@@ -195,18 +205,14 @@ class RetrievalEngine:
         # Dynamic top-k for aggregation queries
         effective_limit = 100 if strat.query_type == "aggregation" else limit
 
-        # 3. Run channels. Set the scope flags on the shared channel instances
-        # and execute them under self._scope_lock so a concurrent recall can't
-        # interleave its scope visibility onto these channels mid-flight. The
-        # worker threads spawned inside _run_channels are joined before the lock
-        # releases, so every channel read sees THIS recall's flags.
-        with self._scope_lock:
-            for ch in (self._semantic, self._bm25, self._entity, self._temporal,
-                       self._hopfield, self._spreading_activation):
-                if ch is not None:
-                    ch.include_global = include_global
-                    ch.include_shared = include_shared
-            ch_results = self._run_channels(query, profile_id, strat)
+        # 3. Run channels. Both scope flags AND extra_disabled_channels travel as
+        # explicit call parameters so concurrent recalls with different flags
+        # cannot corrupt each other.  No lock needed — no shared mutable state.
+        ch_results = self._run_channels(
+            query, profile_id, strat,
+            extra_disabled_channels=extra_disabled_channels,
+            include_global=include_global, include_shared=include_shared,
+        )
         _em("run_channels")
         if profile_hits:
             ch_results["profile"] = profile_hits
@@ -318,6 +324,32 @@ class RetrievalEngine:
                 logger.warning("Entity graph signal enhancement: %s", exc)
 
         _em("expand+entity_enh")
+
+        # T-window: prune candidates to the requested event-time range.
+        # Event times are fetched for the bounded candidate set only (indexed),
+        # then in-range facts are kept — before fact load, so out-of-window facts
+        # are never materialized. T3: when the caller passes no explicit window,
+        # infer one from natural-language scope in the query ("last week").
+        # Safety: an EXPLICIT window is authoritative (honoured even if it empties
+        # the set — the user asked for that scope), but an INFERRED window is
+        # additive and never makes recall worse — if it would empty the results,
+        # fall back to the unwindowed set.
+        _explicit_window = window is not None
+        _window = window if _explicit_window else infer_window_from_query(query)
+        if _window is not None and fused:
+            bounds = parse_window(_window)
+            if bounds is not None:
+                etimes = self._db.get_fact_event_times(
+                    [fr.fact_id for fr in fused], profile_id,
+                )
+                windowed = [
+                    fr for fr in fused
+                    if in_window(etimes.get(fr.fact_id), bounds)
+                ]
+                if windowed or _explicit_window:
+                    fused = windowed
+                _em("time_window")
+
         # 4. Load facts for rerank pool
         pool = min(len(fused), max(effective_limit * 3, 30))
         top = fused[:pool]
@@ -411,8 +443,85 @@ class RetrievalEngine:
             no_confident_match=no_match,
             reranker_applied=reranker_applied,
             reranker_status=reranker_status,
+            # Q2b: thematic context when the top results cluster in one
+            # community. Precomputed summary lookup only — no per-query LLM.
+            community_context=self._community_context(results, profile_id),
         )
 
+    # -- Community context (Wave Q2b) --------------------------------------
+
+    def _community_context(
+        self, results: list[Any], profile_id: str, top_k: int = 8,
+    ) -> dict | None:
+        """Attach the precomputed community summary the top results fall into.
+
+        On-device-safe (market CRIT-1): a single read of the ≤N precomputed
+        community_summaries rows + a membership tally — never a per-query LLM
+        fan-out. Gated: fires only when >=2 of the top results AND >=40% of
+        them belong to one community, so precise factual queries are untouched.
+        Fail-open: any error returns None (recall is never affected).
+        """
+        if not results or not getattr(
+            self._config, "enable_community_context", True,
+        ):
+            return None
+        try:
+            import json
+            from collections import Counter
+
+            rows = [
+                dict(r) for r in self._db.execute(
+                    "SELECT community_id, summary, keywords, fact_ids_json, "
+                    "fact_count FROM community_summaries WHERE profile_id = ?",
+                    (profile_id,),
+                )
+            ]
+            if not rows:
+                return None
+
+            fact_to_cid: dict[str, int] = {}
+            summ_by_cid: dict[int, dict] = {}
+            for r in rows:
+                cid = int(r["community_id"])
+                summ_by_cid[cid] = r
+                try:
+                    for fid in json.loads(r.get("fact_ids_json") or "[]"):
+                        fact_to_cid[str(fid)] = cid
+                except (ValueError, TypeError):
+                    continue
+
+            top_ids = [
+                res.fact.fact_id
+                for res in results[:top_k]
+                if getattr(res, "fact", None) is not None
+            ]
+            tally = Counter(
+                fact_to_cid[fid] for fid in top_ids if fid in fact_to_cid
+            )
+            if not tally:
+                return None
+            best_cid, count = tally.most_common(1)[0]
+            coverage = count / len(top_ids) if top_ids else 0.0
+            if count < 2 or coverage < 0.4:
+                return None
+
+            row = summ_by_cid[best_cid]
+            try:
+                members = json.loads(row.get("fact_ids_json") or "[]")
+            except (ValueError, TypeError):
+                members = []
+            return {
+                "community_id": best_cid,
+                "summary": row.get("summary", ""),
+                "keywords": row.get("keywords", ""),
+                "member_fact_ids": members,
+                "coverage": round(coverage, 3),
+                "matched_results": count,
+            }
+        except Exception as exc:
+            logger.debug("community context skipped (fail-open): %s", exc)
+            return None
+
     # -- Evidence floor (v3.6.6) -------------------------------------------
 
     @staticmethod
@@ -570,32 +679,47 @@ class RetrievalEngine:
         fused: list,
         ch_results: dict[str, list[tuple[str, float]]],
         effective_limit: int,
-        min_per_channel: int = 2,
     ) -> list:
-        """Ensure structure channels (entity_graph) get representation.
-
-        V3.4.11: entity_graph finds valid results but RRF scores them low
-        because they don't overlap with semantic/bm25 results. This interleaves
-        top entity_graph facts into positions 3-4 of the final output instead
-        of appending at the end where they'd never be seen.
+        """Keep strong lexical and structure evidence visible in the result cap.
+
+        A semantic channel with a larger weight can fill a small result limit
+        even when BM25 has an exact, high-signal hit. That broke the
+        ``queryable now`` ingestion contract: a freshly inserted FTS row could
+        exist durably but remain invisible to immediate recall. Reserve one
+        capped slot for a strong BM25 hit and two for a structure channel when
+        such candidates exist, without returning more than ``effective_limit``.
         """
-        structure_channels = ["entity_graph"]
+        channel_minimums = (
+            ("bm25", 1, 0.0),
+            ("entity_graph", 2, 0.0),
+        )
         top_ids = {fr.fact_id for fr in top}
 
         promoted = []
-        for ch_name in structure_channels:
+        for ch_name, minimum, score_floor in channel_minimums:
             ch_items = ch_results.get(ch_name, [])
             if not ch_items:
                 continue
 
-            present = sum(1 for fid, _ in ch_items if fid in top_ids)
-            if present >= min_per_channel:
+            eligible_ids = {
+                fid
+                for fid, score in ch_items
+                if (
+                    float(score) > score_floor
+                    if ch_name == "bm25"
+                    else float(score) >= score_floor
+                )
+            }
+            if not eligible_ids:
+                continue
+
+            present = sum(1 for fid in eligible_ids if fid in top_ids)
+            if present >= minimum:
                 continue
 
-            needed = min_per_channel - present
-            ch_fids = {fid for fid, _ in ch_items}
+            needed = minimum - present
             for fr in fused:
-                if fr.fact_id in ch_fids and fr.fact_id not in top_ids:
+                if fr.fact_id in eligible_ids and fr.fact_id not in top_ids:
                     promoted.append(fr)
                     top_ids.add(fr.fact_id)
                     needed -= 1
@@ -605,10 +729,15 @@ class RetrievalEngine:
         if not promoted:
             return top
 
-        # Append as safety net — with proper RRF weights (strategy.py),
-        # entity_graph facts should already rank naturally in the top-k.
-        # This only fires when they're still missing despite weight boost.
-        return list(top) + promoted
+        selected = promoted[:effective_limit]
+        result = list(top[:effective_limit])
+        free_slots = max(0, effective_limit - len(result))
+        result.extend(selected[:free_slots])
+        remaining = selected[free_slots:]
+        if remaining:
+            keep = max(0, effective_limit - len(remaining))
+            result = result[:keep] + remaining
+        return result[:effective_limit]
 
     # -- Channel execution --------------------------------------------------
 
@@ -628,7 +757,14 @@ class RetrievalEngine:
         return emb
 
     def _run_channels(
-        self, query: str, profile_id: str, strat: QueryStrategy,
+        self,
+        query: str,
+        profile_id: str,
+        strat: QueryStrategy,
+        *,
+        extra_disabled_channels: set[str] | None = None,
+        include_global: bool = False,
+        include_shared: bool = False,
     ) -> dict[str, list[tuple[str, float]]]:
         """Run active retrieval channels.
 
@@ -646,7 +782,9 @@ class RetrievalEngine:
         out: dict[str, list[tuple[str, float]]] = {}
         # Skip channels listed in disabled_channels (ablation support)
         # V3.4.40: union with per-recall extra_disabled set (e.g. --fast skip)
-        disabled = set(self._config.disabled_channels) | getattr(self, "_extra_disabled", set())
+        # V3.4.64: extra_disabled is now a local parameter, not a shared instance
+        # attribute — eliminates the concurrent-recall race condition.
+        disabled = set(self._config.disabled_channels) | set(extra_disabled_channels or ())
 
         # V3.3.4: Embed query ONCE, reuse for semantic + hopfield channels
         q_emb: list[float] | None = None
@@ -688,22 +826,38 @@ class RetrievalEngine:
         if self._semantic is not None and q_emb is not None and "semantic" not in disabled:
             futures["semantic"] = executor.submit(
                 _safe_channel, "semantic",
-                self._semantic.search, q_emb, profile_id, self._config.semantic_top_k,
+                functools.partial(
+                    self._semantic.search,
+                    include_global=include_global, include_shared=include_shared,
+                ),
+                q_emb, profile_id, self._config.semantic_top_k,
             )
         if self._bm25 is not None and "bm25" not in disabled:
             futures["bm25"] = executor.submit(
                 _safe_channel, "bm25",
-                self._bm25.search, query, profile_id, self._config.bm25_top_k,
+                functools.partial(
+                    self._bm25.search,
+                    include_global=include_global, include_shared=include_shared,
+                ),
+                query, profile_id, self._config.bm25_top_k,
             )
         if self._temporal is not None and "temporal" not in disabled:
             futures["temporal"] = executor.submit(
                 _safe_channel, "temporal",
-                self._temporal.search, query, profile_id, self._config.bm25_top_k,
+                functools.partial(
+                    self._temporal.search,
+                    include_global=include_global, include_shared=include_shared,
+                ),
+                query, profile_id, self._config.bm25_top_k,
             )
         if self._hopfield is not None and q_emb is not None and "hopfield" not in disabled:
             futures["hopfield"] = executor.submit(
                 _safe_channel, "hopfield",
-                self._hopfield.search, q_emb, profile_id, self._config.hopfield_top_k,
+                functools.partial(
+                    self._hopfield.search,
+                    include_global=include_global, include_shared=include_shared,
+                ),
+                q_emb, profile_id, self._config.hopfield_top_k,
             )
         if (
             self._spreading_activation is not None
@@ -712,17 +866,33 @@ class RetrievalEngine:
         ):
             futures["spreading_activation"] = executor.submit(
                 _safe_channel, "spreading_activation",
-                self._spreading_activation.search, q_emb, profile_id, self._config.bm25_top_k,
+                functools.partial(
+                    self._spreading_activation.search,
+                    include_global=include_global, include_shared=include_shared,
+                ),
+                q_emb, profile_id, self._config.bm25_top_k,
             )
 
-        # Collect results as channels complete.
+        # Each local channel gets a strict latency budget.  A slow graph walk
+        # must not make an interactive recall wait 30 seconds; completed
+        # channels still participate in fusion and the timeout is observable.
+        channel_timeout_seconds = 1.0
+        # One shared deadline keeps parallel dispatch genuinely bounded.  A
+        # per-future timeout here would serialise the wait and turn five slow
+        # channels into five seconds of UI latency.
+        done, pending = concurrent.futures.wait(
+            futures.values(), timeout=channel_timeout_seconds,
+        )
         for name, fut in futures.items():
+            if fut in pending:
+                logger.warning("Channel %s exceeded %.1fs latency budget", name, channel_timeout_seconds)
+                continue
             try:
-                ch_name, result = fut.result(timeout=30)
+                ch_name, result = fut.result()
                 if result:
                     out[ch_name] = result
             except Exception as exc:
-                logger.warning("Channel %s timed out or failed: %s", name, exc)
+                logger.warning("Channel %s failed: %s", name, exc)
 
         # Apply registered post-retrieval filters (forgetting filter, etc.)
         if hasattr(self, '_registry') and self._registry._filters:
@@ -981,6 +1151,10 @@ class RetrievalEngine:
 
 _CHANNEL_KEYS: tuple[str, ...] = (
     "semantic", "bm25", "entity_graph", "temporal",
+    # hopfield + spreading_activation are real retrieval channels (score
+    # contract v2) with bandit-chosen weights; omitting them here silently
+    # discarded adaptive reranking for multi-hop relational recall.
+    "spreading_activation", "hopfield",
 )
 
 

package/src/superlocalmemory/retrieval/entity_channel.py

@@ -835,13 +835,15 @@ class EntityGraphChannel:
             )[:top_k]
             # Shadow SQLite before accepting a projected answer.  The graph
             # channel has optional PageRank/community enrichments, so exact
-            # score equality is neither required nor useful; result ordering
-            # and membership are the correctness contract.  Any divergence is
-            # recorded and fails closed to canonical SQLite.
+            # Score equality is neither required nor useful; result *membership*
+            # is the correctness contract. Order within the same fact set is
+            # tolerated — requiring identical ordering would fail closed on
+            # every query with score ties, leaving Cozo permanently unused.
+            # Any membership divergence is recorded and fails closed to SQLite.
             sqlite_results = self._search_without_cozo(query, profile_id, top_k)
-            matches = [fact_id for fact_id, _ in cozo_results] == [
+            matches = {fact_id for fact_id, _ in cozo_results} == {
                 fact_id for fact_id, _ in sqlite_results
-            ]
+            }
             record = getattr(self._cozo, "record_shadow_comparison", None)
             if callable(record):
                 record(matches=matches, projected=cozo_results, canonical=sqlite_results)

package/src/superlocalmemory/retrieval/hopfield_channel.py

@@ -88,6 +88,8 @@ class HopfieldChannel:
         query: Any,
         profile_id: str,
         top_k: int = 50,
+        include_global: bool | None = None,
+        include_shared: bool | None = None,
     ) -> list[tuple[str, float]]:
         """Search for facts using Hopfield associative retrieval.
 
@@ -95,6 +97,9 @@ class HopfieldChannel:
             query: Query embedding (list[float] or np.ndarray).
             profile_id: Scope search to this profile.
             top_k: Maximum results to return.
+            include_global: Include global-scope facts. Falls back to the
+                instance attribute when not supplied.
+            include_shared: Include shared-scope facts. Same fallback.
 
         Returns:
             List of (fact_id, score) sorted by score descending.
@@ -104,8 +109,10 @@ class HopfieldChannel:
         if not self._config.enabled:
             return []
 
-        include_global = bool(getattr(self, "include_global", False))
-        include_shared = bool(getattr(self, "include_shared", False))
+        if include_global is None:
+            include_global = bool(getattr(self, "include_global", False))
+        if include_shared is None:
+            include_shared = bool(getattr(self, "include_shared", False))
         try:
             with self._cache_lock:
                 return self._search_inner(

package/src/superlocalmemory/retrieval/reranker.py

@@ -25,9 +25,8 @@ import sys
 import threading
 import time
 import weakref
-from typing import Any
-
 from pathlib import Path
+from typing import Any
 
 from superlocalmemory.infra.data_root import state_path
 from superlocalmemory.storage.models import AtomicFact
@@ -57,9 +56,9 @@ _live_rerankers: set[weakref.ref] = set()
 
 logger = logging.getLogger(__name__)
 
-_IDLE_TIMEOUT_SECONDS = 300  # V3.4.37: 5 min (was 30) — balance cold-start vs RAM.
+_IDLE_TIMEOUT_SECONDS = 1800  # V3.8.1: keep interactive sessions warm.
 # V3.3.12: Configurable via SLM_RERANKER_IDLE_TIMEOUT env var.
-# V3.4.19: Bumped from 120 → 1800 in lock-step with the embedding worker.
+# Low-RAM installations can retain aggressive recycling through the override.
 # Set ``SLM_RERANKER_IDLE_TIMEOUT=120`` + ``slm restart`` to revert.
 _IDLE_TIMEOUT_SECONDS = int(os.environ.get("SLM_RERANKER_IDLE_TIMEOUT", _IDLE_TIMEOUT_SECONDS))
 _SUBPROCESS_RESPONSE_TIMEOUT = 15  # v3.4.52: 15s (was 180s). Long timeout blocked the