switchroom 0.21.18 → 0.21.20

package/vendor/hindsight-memory/scripts/recall.py

@@ -92,6 +92,13 @@ _IMPORT_ELAPSED_SECONDS = time.monotonic() - _IMPORT_START_MONOTONIC
 
 LAST_RECALL_STATE = "last_recall.json"
 RECALL_CACHE_STATE = "recall_cache.json"
+# M4 P-REC F3 — the last prefetch-buffer sentinel token this session has
+# already CONSUMED, keyed by session_id. Persisted across turns so a buffer
+# produced at turn N and consumed at N+1 is not re-served as "fresh" at
+# N+2..N+k when no newer sentinel has landed (the stale-buffer-served-as-fresh
+# class). A dict {session_id: token}; capped like the other per-session state.
+PREFETCH_CONSUMED_STATE = "prefetch_consumed.json"
+_PREFETCH_CONSUMED_MAX_SESSIONS = 10000
 
 # Switchroom hindsight-leverage A3 — label for the directives fetch slot in the
 # parallel fan-out. Distinct from any bank_id (banks can't start with "__") so a
@@ -500,12 +507,6 @@ def _emit_cached_context(context: str) -> None:
     )
 
 
-_PREFETCH_DEGRADED_NOTICE = (
-    "⏳ prefetch not ready and no prior recall is cached for this session — "
-    "proceeding without injected memory this turn."
-)
-
-
 def stale_recall_notice(memories_context: str) -> str:
     """Wrap a PRIOR turn's cached memories-only block in an explicit
     staleness marker for the M4 prefetch-buffer fallback path.
@@ -528,6 +529,100 @@ def stale_recall_notice(memories_context: str) -> str:
     )
 
 
+def _read_consumed_token(session_id: str) -> "int | None":
+    """Return the last prefetch sentinel token this session already consumed,
+    or None if none is recorded. Failure-tolerant — any read/parse error
+    returns None (treat as "nothing consumed yet", the fail-open direction that
+    at worst re-serves once, never suppresses forever)."""
+    state = read_state(PREFETCH_CONSUMED_STATE, {}) or {}
+    if not isinstance(state, dict):
+        return None
+    token = state.get(session_id)
+    return int(token) if isinstance(token, (int, float)) else None
+
+
+def _write_consumed_token(session_id: str, token: int) -> None:
+    """Persist `token` as the last sentinel this session has consumed. Bounded
+    to `_PREFETCH_CONSUMED_MAX_SESSIONS` entries. Best-effort — write_state
+    never raises."""
+    state = read_state(PREFETCH_CONSUMED_STATE, {}) or {}
+    if not isinstance(state, dict):
+        state = {}
+    state[session_id] = int(token)
+    if len(state) > _PREFETCH_CONSUMED_MAX_SESSIONS:
+        # Drop the numerically-smallest tokens (oldest producers) first.
+        for k in sorted(
+            state.keys(),
+            key=lambda k: state[k] if isinstance(state[k], (int, float)) else 0,
+        )[: len(state) - _PREFETCH_CONSUMED_MAX_SESSIONS]:
+            state.pop(k, None)
+    write_state(PREFETCH_CONSUMED_STATE, state)
+
+
+def curate_recall_results(results, config, bank_id):
+    """M4 F5 — the SHARED recall curation pipeline, so the async prefetch
+    producer (`prefetch.py`) can never drift from the guarantees the
+    synchronous recall path enforces.
+
+    Applies, in order, the exact same module-level primitives the synchronous
+    path composes inline: drop demote-tagged memories (`_is_demoted_memory`),
+    sort by engine relevance (`_sort_by_final_score`), apply the optional
+    absolute score floor when it is configured for ALL turns
+    (`_filter_by_min_score`; the `degraded`-scope floor is a synchronous-path
+    concept the speculative single-bank prefetch cannot evaluate, so it is
+    honoured here only when `recallMinScoreScope` is `all`, matching what the
+    sync path does on a healthy own-bank turn), and finally the
+    `recallMaxMemories` head cap with per-bank slot reservation
+    (`_reserve_bank_slots`).
+
+    Returns the curated result list. Never mutates the caller's list in a way
+    that changes its length before returning (it rebinds internally). Pure —
+    no I/O, no network — so it is safe to call from the producer hook.
+    """
+    if not results:
+        return results
+
+    # 1) demote-from-recall drop (Switchroom #432 4.4) — before any cap so the
+    #    cap fills with non-demoted hits.
+    results = [m for m in results if not _is_demoted_memory(m)]
+
+    # 2) relevance sort (Phase-1 bank-starvation fix) — in place on our copy.
+    _sort_by_final_score(results)
+
+    # 3) optional absolute score floor (#3837). The sync path binds this on
+    #    `all` scope always, and on `degraded` scope only when the own-bank
+    #    read degraded. A prefetch is a healthy speculative read (no degraded
+    #    signal), so we bind only the `all` scope here — identical to the sync
+    #    path's decision on a non-degraded turn.
+    min_score_floor = config.get("recallMinScore", 0.0)
+    if isinstance(min_score_floor, bool) or not isinstance(min_score_floor, (int, float)):
+        min_score_floor = 0.0
+    min_score_floor = float(min_score_floor)
+    min_score_scope = config.get("recallMinScoreScope", "degraded")
+    if min_score_floor > 0 and min_score_scope == "all":
+        results, _dropped = _filter_by_min_score(results, min_score_floor)
+
+    # 4) recallMaxMemories head cap + per-bank slot reservation. `_reserve_
+    #    bank_slots` is a passthrough when cap<=0 or the set already fits, and
+    #    with the fleet-default 0/0 floors on a single-bank prefetch it reduces
+    #    to a plain head-slice — the same slice the sync path takes.
+    recall_max_memories = config.get("recallMaxMemories", 0)
+    if (
+        isinstance(recall_max_memories, int)
+        and recall_max_memories > 0
+        and len(results) > recall_max_memories
+    ):
+        results, _own, _add = _reserve_bank_slots(
+            results,
+            recall_max_memories,
+            bank_id,
+            config.get("recallOwnBankMinSlots", 0),
+            config.get("recallAdditionalBankMinSlots", 0),
+        )
+
+    return results
+
+
 def _handle_prefetch_buffer(config: dict, hook_input: dict, prompt: str) -> bool:
     """M4 P-REC Fix C (consumer side) — join the Stop-hook producer's
     prefetch buffer for this session instead of running recall
@@ -535,10 +630,12 @@ def _handle_prefetch_buffer(config: dict, hook_input: dict, prompt: str) -> bool
 
     Gated entirely by `config.get("memoryPrefetchEnabled", False)` at the
     caller; this function assumes the flag is already on. Returns True iff
-    it emitted an `additionalContext` payload (fresh hit, stale fallback, or
-    the explicit degraded notice) and the caller should return without
-    running the synchronous path. Returns False on a clean no-op miss (flag
-    effectively off / nothing to say) so the caller falls through.
+    it emitted an `additionalContext` payload (fresh buffer hit, stale
+    fallback, or a directives-only block) and the caller should return
+    without running the synchronous path. Returns False on a miss with
+    nothing to serve — cold start, no fresh buffer, no stale prior recall,
+    no directives — so the caller falls through to SYNCHRONOUS recall (M4
+    F4: the first turn of a fresh/post-restart session must still recall).
 
     Never raises past this function's own boundary in normal operation —
     every internal step is wrapped so a bug here degrades to "fall through
@@ -547,6 +644,13 @@ def _handle_prefetch_buffer(config: dict, hook_input: dict, prompt: str) -> bool
     """
     session_id = hook_input.get("session_id") or "unknown"
 
+    # M4 F3 — the sentinel token this session has ALREADY consumed on a prior
+    # turn. Threaded into both `poll_for_sentinel` and `read_if_fresh` so a
+    # buffer produced at turn N and consumed at N+1 is treated as STALE (not
+    # re-served as fresh) at N+2..N+k until a strictly-newer sentinel lands.
+    # None on a cold session (nothing consumed yet).
+    last_consumed = _read_consumed_token(session_id)
+
     # Cold-start short-circuit (red-team MAJOR finding): if this session has
     # NEVER produced a sentinel, polling the full cap on every single
     # session-open turn would cost ~the poll cap on every fresh session —
@@ -556,9 +660,24 @@ def _handle_prefetch_buffer(config: dict, hook_input: dict, prompt: str) -> bool
         debug_log(config, "Prefetch buffer: no sentinel ever written for this session, cold-start skip")
     else:
         cap_ms = int(config.get("memoryPrefetchPollCapMs", 400))
-        recall_buffer.poll_for_sentinel(session_id, last_consumed_token=None, cap_ms=cap_ms)
-
-    payload, _token = recall_buffer.read_if_fresh(session_id, last_consumed_token=None)
+        recall_buffer.poll_for_sentinel(session_id, last_consumed_token=last_consumed, cap_ms=cap_ms)
+
+    payload, _token = recall_buffer.read_if_fresh(session_id, last_consumed_token=last_consumed)
+
+    # #4778 — TOPIC-RELEVANCE GUARD. `read_if_fresh` only proves the buffer is
+    # FRESH (a strictly-newer sentinel this session), never that it is ON TOPIC:
+    # the producer built it from turn N's last human prompt, so on a topic PIVOT
+    # a perfectly-fresh buffer holds the WRONG-topic memories. Gate the join on
+    # topical similarity between the buffered query and turn N+1's ACTUAL prompt
+    # BEFORE the directive fetch below; a mismatch marks the token consumed (so
+    # it is never reconsidered) and falls through to SYNCHRONOUS recall — which
+    # fetches correct, current-topic memories AND re-injects directives itself.
+    # This is layered ON TOP of the F3 freshness machinery, not a replacement.
+    if payload is not None and not _prefetch_topic_matches(prompt, payload.get("query", ""), config):
+        if _token is not None:
+            _write_consumed_token(session_id, _token)
+        debug_log(config, "Prefetch buffer: topic mismatch vs current prompt, falling through to synchronous recall")
+        return False
 
     # Directives stay on the synchronous, always-fresh path (M3 rule) even
     # in the fast path — fetched here directly, never from the buffer.
@@ -584,6 +703,12 @@ def _handle_prefetch_buffer(config: dict, hook_input: dict, prompt: str) -> bool
         directives_block = None
 
     if payload is not None:
+        # M4 F3 — record this token as consumed BEFORE emitting, so the next
+        # turn's `read_if_fresh` rejects the same buffer as stale unless the
+        # producer has since written a strictly-newer sentinel. `_token` is the
+        # sentinel token `read_if_fresh` just validated as fresh.
+        if _token is not None:
+            _write_consumed_token(session_id, _token)
         memories_block = payload.get("context") or ""
         parts = [b for b in (_dir_notice, directives_block, memories_block) if b]
         if not parts:
@@ -607,11 +732,15 @@ def _handle_prefetch_buffer(config: dict, hook_input: dict, prompt: str) -> bool
         _emit_cached_context("\n\n".join([b for b in (_dir_notice, directives_block) if b]))
         return True
 
-    # Nothing fresh, nothing stale, nothing cached — say so explicitly
-    # rather than silently emitting no context (so a degraded turn is
-    # legible, matching the #3619 degraded-disclosure precedent).
-    _emit_cached_context(_PREFETCH_DEGRADED_NOTICE)
-    return True
+    # M4 F4 — nothing fresh, nothing stale, nothing cached (the cold-start /
+    # first-turn / no-prior-recall case). Return False so the caller runs the
+    # SYNCHRONOUS recall path, which is exactly the path that should run on a
+    # turn where no prefetch could possibly exist yet. Emitting a degraded
+    # notice and returning True here (the previous behaviour) short-circuited
+    # sync recall and handed a fresh/post-restart session ZERO memories plus a
+    # degraded banner on its very first turn — the opposite of the docstring's
+    # "a miss falls through to sync recall" contract.
+    return False
 
 
 def _emit_directives_only(config: dict, hook_input: dict) -> None:
@@ -774,6 +903,78 @@ def _overlap_tokens(text) -> set:
     return out
 
 
+_PREFETCH_TOPIC_OVERLAP_DEFAULT = 0.3
+
+# #4778 review MAJOR — short-prompt Jaccard floor. Jaccard is probabilistic, not
+# absolute, on SHORT prompts: two ~2-content-token turns that share ONE incidental
+# token while EACH carries a divergent token give 1/3 = 0.333 >= 0.30 and wrongly
+# clear the ratio bar ("call mom" buffer served on a "call ended" turn — the pivot
+# signature). When either token set is small (fewer than ``MIN_SMALL_SET_TOKENS``)
+# such a partial overlap must supply at least ``_PREFETCH_MIN_SMALL_SET_INTERSECTION``
+# shared tokens, not just clear the ratio.
+#
+# The floor fires ONLY on that divergent-on-both-sides shape. It deliberately
+# EXEMPTS full containment (``intersection == min(|A|, |B|)`` — the smaller set is
+# wholly shared: identical or subset/narrowing queries like "decide" vs "decide",
+# or "restart klanker" vs "restart the klanker agent"), which is a legitimate
+# topical match regardless of brevity and must still be served. Long prompts (both
+# sides at or above ``MIN_SMALL_SET_TOKENS``) skip the floor entirely and use the
+# Jaccard ratio exactly as before. The floor only ever ANDs a stricter condition
+# onto the ratio — it can reject, never serve.
+MIN_SMALL_SET_TOKENS = 3
+_PREFETCH_MIN_SMALL_SET_INTERSECTION = 2
+
+
+def _prefetch_topic_matches(prompt, buffered_query, config) -> bool:
+    """#4778 — True iff turn N+1's ``prompt`` is topically close enough to the
+    ``buffered_query`` the M4 producer used to build the warm buffer.
+
+    Jaccard overlap (``|A∩B| / |A∪B|``) on ``_overlap_tokens`` — the same
+    stop-word-stripped, digit-preserving, dependency-free tokenizer the
+    transcript-fallback keyword match uses (unicode-safe via ``str.isalnum``,
+    characterised in ``tests/test_overlap_tokens.py``) — against
+    ``memoryPrefetchMinTopicOverlap`` (default 0.3). Jaccard, not the overlap
+    coefficient: both score 0 on a true pivot (disjoint content tokens), so the
+    correctness guarantee is identical, but Jaccard cannot be driven to 1.0 by a
+    single incidental shared token in a short prompt — it biases the residual
+    error toward a harmless false-MISS (sync recall, slower) rather than a
+    false-HIT (wrong-topic memories, the bug).
+
+    Fail-safe: an empty token set on EITHER side — a contentless prompt, or a
+    legacy/torn buffer whose ``query`` is ``""`` — returns False, so the caller
+    falls through to synchronous recall. A miss only ever costs latency; it can
+    never serve wrong-topic memories. Both sides are stripped of the ``<channel>``
+    envelope first, matching the producer's own query derivation.
+    """
+    now_tokens = _overlap_tokens(strip_channel_envelope(prompt or ""))
+    buf_tokens = _overlap_tokens(strip_channel_envelope(buffered_query or ""))
+    if not now_tokens or not buf_tokens:
+        return False
+    intersection = len(now_tokens & buf_tokens)
+    if intersection == 0:
+        return False
+    # Short-prompt floor (#4778 review MAJOR): when either side is small, a single
+    # incidental shared token must not clear the guard on the Jaccard ratio alone.
+    # Exempt full containment (intersection == smaller set) — identical/subset
+    # queries are a legitimate topical match, never the divergent-on-both-sides
+    # pivot this floor targets. ANDed with the ratio check below: stricter, never
+    # looser, and long prompts (both sides >= MIN_SMALL_SET_TOKENS) are untouched.
+    min_set = min(len(now_tokens), len(buf_tokens))
+    if min_set < MIN_SMALL_SET_TOKENS \
+            and intersection < min_set \
+            and intersection < _PREFETCH_MIN_SMALL_SET_INTERSECTION:
+        return False
+    union = len(now_tokens | buf_tokens)
+    threshold = config.get("memoryPrefetchMinTopicOverlap", _PREFETCH_TOPIC_OVERLAP_DEFAULT)
+    try:
+        threshold = float(threshold)
+    except (TypeError, ValueError):
+        threshold = _PREFETCH_TOPIC_OVERLAP_DEFAULT
+    if not (0.0 <= threshold <= 1.0):
+        threshold = _PREFETCH_TOPIC_OVERLAP_DEFAULT
+    return (intersection / union) >= threshold
+
+
 def _result_final_score(m) -> float:
     """Return a result's engine relevance score (`scores.final`).
 

package/vendor/hindsight-memory/scripts/tests/test_prefetch_invalidation.py

@@ -278,7 +278,9 @@ class NoDuplicateInjectionTests(InvalidationBase):
         config = self._config(prefetch_enabled=True)
         marker = "- unique-fact-abc123 decided at standup"
 
-        recall_buffer.write_buffer(SESSION, marker, {})
+        # #4778 — on-topic with the `_consume` prompt ("what did we decide") so
+        # the topic guard passes and this stays a genuine warm-buffer hit.
+        recall_buffer.write_buffer(SESSION, marker, {}, query="what did we decide")
         recall_buffer.write_sentinel(SESSION)
 
         # Turn N+1: consumer injects the buffered block once.
@@ -311,7 +313,10 @@ class ReplyPathDoesNotBlockOnRecallTests(InvalidationBase):
 
     def test_fresh_buffer_served_without_calling_recall(self):
         config = self._config(prefetch_enabled=True)
-        recall_buffer.write_buffer(SESSION, "- a prefetched memory xyz", {})
+        # #4778 — on-topic with the `_consume` prompt so the topic guard passes;
+        # the point of THIS test is that a matched fresh buffer never blocks on a
+        # synchronous recall, so the query must clear the guard.
+        recall_buffer.write_buffer(SESSION, "- a prefetched memory xyz", {}, query="what did we decide")
         recall_buffer.write_sentinel(SESSION)
 
         class _ExplodingRecallClient:

package/vendor/hindsight-memory/scripts/tests/test_prefetch_pipeline.py

@@ -147,7 +147,7 @@ class CallOrderTests(PrefetchPipelineBase):
             order.append("retain")
             return {"status": "ok"}
 
-        def _spy_write_buffer(session_id, context, telemetry=None):
+        def _spy_write_buffer(session_id, context, telemetry=None, query=None):
             order.append("buffer")
 
         def _spy_write_sentinel(session_id):
@@ -233,14 +233,105 @@ class KillSwitchOffTests(PrefetchPipelineBase):
 
 
 class JunkGateTests(PrefetchPipelineBase):
-    def test_task_notification_turn_is_skipped(self):
-        _write_transcript(self.transcript_path, [("user", "irrelevant")])
-        with patch("prefetch.retain_module.run_retain", side_effect=AssertionError("must not retain a task-notification turn")):
-            wrote = prefetch.run_prefetch(
-                self._hook_input("<task-notification>done</task-notification>"), self._config()
-            )
+    def test_task_notification_in_transcript_skips_before_retain(self):
+        # F6 (regression guard): a real Stop-hook input carries NO `prompt`
+        # field — only session_id / transcript_path / stop_hook_active — so the
+        # junk gate MUST derive from the transcript's last human turn. When that
+        # turn is a `<task-notification>` envelope, prefetch skips ENTIRELY:
+        # no delta retain, no recall, no buffer. This fails on the pre-fix code,
+        # whose `hook_input.get("prompt")` gate is permanently empty on real
+        # input and therefore lets retain run on a task-notification turn.
+        _write_transcript(
+            self.transcript_path,
+            [("user", "<task-notification>sub-agent done</task-notification>")],
+        )
+        real_stop_input = {
+            "session_id": SESSION,
+            "transcript_path": self.transcript_path,
+            "stop_hook_active": True,
+            "cwd": "/tmp",
+        }  # deliberately NO `prompt` / `user_prompt` key — mirrors real input
+        retain_spy = mock.Mock(return_value={"status": "ok"})
+        with patch("prefetch.retain_module.run_retain", retain_spy):
+            wrote = prefetch.run_prefetch(real_stop_input, self._config())
         self.assertFalse(wrote)
+        retain_spy.assert_not_called()
         self.assertFalse(os.path.isfile(recall_buffer._buffer_path(SESSION)))
+        self.assertFalse(os.path.isfile(recall_buffer._sentinel_path(SESSION)))
+
+    def test_gate_keys_off_transcript_not_a_phantom_prompt_field(self):
+        # The mirror of the above: a genuine human turn in the transcript with
+        # a STRAY `prompt="<task-notification>"` on the hook input (the shape the
+        # old harness faked). The gate must key off the transcript (a real turn)
+        # and PROCEED, proving it no longer reads `hook_input["prompt"]`. Fails
+        # on the pre-fix code, which reads the phantom field and wrongly skips.
+        _write_transcript(self.transcript_path, [("user", "we shipped the release on friday")])
+        client = mock.Mock()
+        client.recall.side_effect = lambda *a, **kw: {"results": [
+            {"text": "we shipped the release on friday", "type": "fact",
+             "mentioned_at": "2026-01-01", "id": "r1", "scores": {"final": 0.9}},
+        ]}
+        hook_input = {
+            "session_id": SESSION,
+            "transcript_path": self.transcript_path,
+            "prompt": "<task-notification>phantom</task-notification>",
+            "cwd": "/tmp",
+        }
+        with patch("prefetch.retain_module.run_retain", return_value={"status": "ok"}), \
+                patch("prefetch.HindsightClient", return_value=client), \
+                patch("prefetch.get_api_url", return_value="http://fake"):
+            wrote = prefetch.run_prefetch(hook_input, self._config())
+        self.assertTrue(wrote)
+        self.assertTrue(os.path.isfile(recall_buffer._buffer_path(SESSION)))
+
+
+class CurationSharingTests(PrefetchPipelineBase):
+    """F5 — the producer must run recalled candidates through recall's shared
+    curation pipeline (demote-drop + `recallMaxMemories` cap) BEFORE buffering,
+    so the buffer path cannot inject what the synchronous path would filter."""
+
+    def _run_with_results(self, results, config):
+        _write_transcript(self.transcript_path, [("user", "what do you remember about deploys")])
+        client = mock.Mock()
+        client.recall.side_effect = lambda *a, **kw: {"results": [dict(m) for m in results]}
+        with patch("prefetch.retain_module.run_retain", return_value={"status": "ok"}), \
+                patch("prefetch.HindsightClient", return_value=client), \
+                patch("prefetch.get_api_url", return_value="http://fake"):
+            wrote = prefetch.run_prefetch(self._hook_input("what do you remember about deploys"), config)
+        buffered = ""
+        if wrote:
+            payload = recall_buffer._read_buffer_payload(SESSION) or {}
+            buffered = payload.get("context") or ""
+        return wrote, buffered
+
+    def test_demote_tagged_memory_is_dropped_before_buffering(self):
+        results = [
+            {"text": "keep me visible", "type": "fact", "mentioned_at": "2026-01-01",
+             "id": "k1", "scores": {"final": 0.9}},
+            {"text": "secret demoted note", "type": "fact", "mentioned_at": "2026-01-01",
+             "id": "d1", "scores": {"final": 0.8}, "tags": ["demote-from-recall"]},
+        ]
+        _wrote, buffered = self._run_with_results(results, self._config())
+        self.assertIn("keep me visible", buffered)
+        self.assertNotIn(
+            "secret demoted note", buffered,
+            "a demote-from-recall memory must never reach the prefetch buffer",
+        )
+
+    def test_candidate_set_above_cap_is_truncated_before_buffering(self):
+        results = [
+            {"text": f"candidate {i}", "type": "fact", "mentioned_at": "2026-01-01",
+             "id": f"m{i}", "scores": {"final": 1.0 - i * 0.01}}
+            for i in range(12)
+        ]
+        config = self._config()
+        config["recallMaxMemories"] = 8
+        _wrote, buffered = self._run_with_results(results, config)
+        injected = sum(1 for i in range(12) if f"candidate {i}" in buffered)
+        self.assertEqual(
+            injected, 8,
+            "the prefetch buffer must honour recallMaxMemories, same as the sync path",
+        )
 
 
 if __name__ == "__main__":