switchroom 0.19.24 → 0.19.26

package/vendor/hindsight-memory/scripts/recall.py

@@ -39,14 +39,21 @@ Exit codes:
       stderr and non-zero exit. Existing behaviour.
 """
 
-import hashlib
-import json
-import os
-import re
-import socket
-import sys
 import time
-import urllib.error
+
+# Taken before anything else is imported, so `_IMPORT_ELAPSED_SECONDS` below
+# captures the real cost of loading this hook's dependencies. That spend is
+# charged against the UserPromptSubmit ceiling (see `HOOK_CEILING_SECONDS`) —
+# `recall_start_monotonic` is taken well into main() and cannot see it.
+_IMPORT_START_MONOTONIC = time.monotonic()
+
+import hashlib  # noqa: E402
+import json  # noqa: E402
+import os  # noqa: E402
+import re  # noqa: E402
+import socket  # noqa: E402
+import sys  # noqa: E402
+import urllib.error  # noqa: E402
 
 sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
 
@@ -58,8 +65,10 @@ from lib.content import (
     compose_recall_query,
     format_current_time,
     format_memories,
+    shape_recall_query,
     strip_channel_envelope,
     strip_memory_tags,
+    tokenize_for_bm25,
     truncate_recall_query,
 )
 from lib.daemon import get_api_url
@@ -73,6 +82,11 @@ from lib.gateway_ipc import extract_chat_id_from_prompt, extract_topic_from_prom
 from lib.parallel_recall import run_parallel
 from lib.state import read_state, write_state
 
+# Cost of everything above, charged against the hook ceiling (see
+# `HOOK_CEILING_SECONDS`). Measured, not estimated: it is dominated by
+# `lib.client` pulling in urllib/ssl and by `lib.content` on cold page cache.
+_IMPORT_ELAPSED_SECONDS = time.monotonic() - _IMPORT_START_MONOTONIC
+
 LAST_RECALL_STATE = "last_recall.json"
 RECALL_CACHE_STATE = "recall_cache.json"
 
@@ -396,140 +410,42 @@ def _is_demoted_memory(memory) -> bool:
     return False
 
 
-# Switchroom #475 — lexical-overlap relevance gate.
-#
-# Hindsight's HTTP recall API DOES return per-result relevance scores
-# (`RecallResult.scores.final`, plus `.semantic`/`.keyword`/`.reranker`);
-# the merged multi-bank set is now sorted by `scores.final` before the
-# `recallMaxMemories` cap (see the sort just before the cap in
-# process_recall) so the most relevant memories survive the head-slice
-# regardless of which bank they came from. This gate is a *complementary*,
-# opt-in absolute precision floor: `scores.final` is a relative rank that
-# still orders weakly-matching memories rather than excluding them, so on a
-# low-relevance prompt the top-N could still be low-signal. The lexical
-# overlap between the user's query terms and each memory's text terms is a
-# rank-independent absolute measure that drops memories below a
-# configurable threshold outright — something the relative sort does not do.
-#
-# --- switchroom #3541: the metric is CONTAINMENT, not Jaccard ---
-#
-# This gate originally scored with Jaccard similarity,
-# `|Q ∩ M| / |Q ∪ M|`. That is the wrong metric here, because the two
-# sides have wildly asymmetric lengths: the recall query is the whole
-# prior-context preamble (production p50 778 chars) while a memory is a
-# single fact (tens of tokens). The union term is dominated by |Q|, so the
-# SAME memory with the SAME real overlap scores lower purely because the
-# prompt was longer. Query length, not relevance, decided the outcome.
-#
-# Measured on production recall telemetry (1548 rows, 646 of which put
-# candidates through the gate, ts >= 2026-07-18, all fleet agents'
-# `recall_log.jsonl`), at the fleet default threshold of 0.10:
-#
-#   query_chars    rows   candidates   survived   survival%   zero-result%
-#      0- 200        30        1364          99       7.3%          23.3%
-#    200- 400       110        4199         238       5.7%          17.3%
-#    400- 600       143        5305         293       5.5%          72.7%
-#    600- 750       213        7288          67       0.9%          93.0%
-#    750-1000       150        4968          70       1.4%          90.0%
-#
-# 22447 of 23124 candidate memories (97.1%) were discarded, and the
-# discard rate rose monotonically with prompt length — the signature of
-# the union-term artifact, not of a relevance judgement. Where the gate
-# actually got to run, it was decisive: of the 526 turns that had at least
-# one bank return successfully, 440 (83.7%) still delivered zero memories.
-#
-# How much of the empty-recall problem is this? Roughly a third — NOT all
-# of it. Partitioning the zero-result recalls by why they were empty:
-#
-#   >=1 bank returned OK, and overlap_dropped > 0   448  (~32%)  <- this gate
-#   ALL banks timed out or errored                  565  (~41%)  <- the deadline
-#   >=1 bank returned OK, nothing dropped             7
-#
-# So the 8s outer deadline accounts for MORE empty recalls than the gate
-# does, and it is still live after this change: `deadline_hit` is set on
-# 991 of 18656 logged recalls and the p50 `total_elapsed_ms` on the
-# deadline-hitting rows sits at the 8s ceiling. Fixing the metric does not
-# close #3541's latency half — see the reranker analysis in that issue.
-# This change fixes the gate; it does not fix the timeout.
-#
-# The correct metric for asymmetric-length comparison is containment:
-# `|Q ∩ M| / |M|` — "what fraction of THIS MEMORY's terms are present in
-# the prompt", invariant to how much unrelated preamble the prompt
-# carries. (Not `min(|Q|, |M|)`; see `containment_overlap` for why the
-# textbook overlap coefficient's short-prompt regime flip is undesirable
-# here.)
+# Tokenizer shared by the transcript fallback (`_build_transcript_fallback`).
 #
-# --- what the 0.10 threshold does, and does not, buy ---
+# --- history: the removed lexical-overlap recall gate (#475, #3541, #3761) ---
 #
-# Be honest about the retained default: at 0.10, containment is close to a
-# PASSTHROUGH. Because the denominator is the memory's own token count, a
-# memory of <= 10 content tokens clears 0.10 on a SINGLE shared word. Only
-# longer memories are meaningfully filtered (a 40-token memory still needs
-# 4 shared terms). Measured on the production-shaped prompt used in the
-# regression tests, a wholly off-topic memory that merely reuses words
-# from the prior-context preamble scores 0.857 — HIGHER than the genuinely
-# relevant memory at 0.571.
+# These tokens used to feed a `recallMinOverlap` gate that ran between the
+# engine's reranker and the `recallMaxMemories` head-slice, dropping any
+# candidate whose containment overlap with the prompt fell below a threshold.
+# It was removed outright — no replacement floor — after measurement showed it
+# was pure loss:
 #
-# That inversion is the reason the threshold is NOT being raised to
-# compensate. Raising it would drop the relevant memory before the
-# off-topic one, because on a preamble-heavy prompt containment is not
-# monotone in relevance. Lexical overlap cannot make that distinction at
-# any threshold; the engine's reranker can, and does.
+#   * On healthy production rows (non-timeout, non-error, ts >= 2026-07-20,
+#     n=212 across the fleet) it discarded 6026 of 7549 post-reranker
+#     candidates — 79.8% fleet-wide, 94.4% for overlord, 91.0% for klanker.
+#   * Replaying 330 real logged queries against the live engine, the gate
+#     dropped the engine's OWN top-ranked candidate on 31.2% of queries. It
+#     was not acting as a floor; it was acting as a rival, worse ranker.
+#   * It filtered no measurable noise. The rate at which the best injected
+#     memory scored below 1e-3 was 27.0% with the gate and 28.2% with no gate
+#     at all — inside sampling noise. It cost a third of top hits and bought
+#     nothing.
 #
-# The threshold was nevertheless re-derived from data rather than inherited
-# from the Jaccard era. 31 real production queries (reconstructed from
-# transcripts with the hook's own compose+truncate, validated by matching
-# 320 of them verbatim against logged recall rows) were replayed against
-# the live engine; the 202 returned candidates were rescored offline:
+# The root cause is in the tokenizer below: it keeps only alphabetic tokens of
+# length > 1, so digits, identifiers, version numbers, PR numbers, file paths
+# and short symbols are invisible to it. For a fleet whose prompts are mostly
+# identifiers that is close to worst case.
 #
-#   containment    survival%   zero-result%   mean kept/query
-#      0.05           99.5%          0.0%           6.5
-#      0.10           86.1%          6.5%           5.6   <- shipped
-#      0.20           65.8%         41.9%           4.3
-#      0.30           58.4%         41.9%           3.8
-#      0.40           52.5%         54.8%           3.4
+# No score floor replaced it. `scores.final` is NOT calibrated across queries —
+# the engine's own docs state a clearly-relevant match may score ~0.001 while
+# ranked first, and freed slots are not backfilled. Measured over the same 330
+# replays, every candidate floor value had `top1lost% == zero%` exactly: a
+# floor never trims a bad tail, it only empties the whole result set. A floor
+# at 0.001 would take zero-result recalls from 5.8% to 28.2%; at 0.05, to
+# 40.6%. That re-creates #3541. `min_scores` is deliberately left unset.
 #
-# (Jaccard @0.10 on the same candidates: 70.3% survival, 32.3%
-# zero-result — i.e. the fix cuts empty recalls from ~1 turn in 3 to
-# ~1 in 15 on this sample.)
-#
-# 0.30 and 0.40 leave 42-55% of turns with NO memories at all — they
-# re-create the exact failure #3541 is about, for a precision gain the
-# inversion above says is illusory. 0.10 is the shipped value.
-#
-# Sample caveat, stated plainly: these are single-bank (overlord) replays
-# and the engine returned ~6.5 already-top-ranked candidates per query,
-# not the ~35-candidate pools seen in the fleet logs. The ABSOLUTE
-# survival percentages are therefore optimistic versus production; the
-# RANKING of the thresholds, and the zero-result cliff above 0.10, are the
-# load-bearing results.
-#
-# So the honest statement of the design after #3541 is: the effective
-# precision control is the engine rerank plus the `recallMaxMemories`
-# head-slice, and this gate is a cheap floor that removes only candidates
-# with (near-)zero lexical relationship to the prompt. It is no longer
-# doing the job the #475 note above describes — "on a low-relevance prompt
-# the top-N could still be low-signal" is a real concern that this gate
-# does not actually address. Doing so needs a relevance-score floor
-# (`scores.final`), not a lexical one; that is deliberately out of scope
-# here and wants its own measured change.
-#
-# This is safe to run permissively: the gate is a FLOOR, not a ranker.
-# `_sort_by_final_score` orders the survivors by the engine's reranked
-# relevance score immediately afterwards, and only then does
-# `recallMaxMemories` head-slice. So admitting more candidates cannot
-# lower the quality of what is injected — it can only give the
-# score-sort a non-empty set to choose the top-N from. And because
-# `|Q ∩ M| / |M| >= |Q ∩ M| / |Q ∪ M|` always, at a fixed threshold this
-# metric admits a superset of what the deployed Jaccard gate admits: no
-# memory that survives in production today can be dropped by this change.
-#
-# Threshold default is 0.0 (disabled) so the gate is opt-in initially.
-# Operators tune via `memory.recall.min_overlap` in switchroom.yaml or
-# `HINDSIGHT_RECALL_MIN_OVERLAP=0.15` env. Telemetry surfaces the dropped
-# count via the existing recall_log.jsonl (#432 4.3) under
-# `overlap_dropped`, so the gate's effect is observable per turn from
-# `switchroom memory recall-log <agent>`.
+# Precision now rests where it belongs: the engine's rerank ordering, the
+# `_sort_by_final_score` merge sort, and the `recallMaxMemories` head-slice.
 #
 # A small English stop-word set is removed from both sides before the
 # overlap is computed — common-word coincidence is not a real signal.
@@ -579,78 +495,6 @@ def _overlap_tokens(text) -> set:
     return out
 
 
-def containment_overlap(query: str, memory_text: str) -> float:
-    """Containment of the MEMORY in the query, after stop-word + punctuation
-    stripping: ``|Q ∩ M| / |M|``.
-
-    Returns a float in [0.0, 1.0]. Empty/degenerate inputs return 0.0 —
-    it's safer to drop than retain when we can't compute.
-
-    Unlike Jaccard (`|Q ∩ M| / |Q ∪ M|`, used until switchroom #3541) this
-    is INVARIANT TO QUERY LENGTH. The recall query is a long prior-context
-    preamble and a memory is a short fact; dividing by the union made the
-    score collapse as the prompt grew, so the gate discarded 97.1% of
-    already-reranked candidates and did so monotonically in prompt length.
-    Dividing by the memory asks the question the gate actually means: what
-    fraction of this memory's terms appear in the prompt. See the design
-    note above `_OVERLAP_STOPWORDS` for the production measurement.
-
-    That invariance is specifically against growth of NON-OVERLAPPING
-    preamble in the query — the failure mode #3541 hit — and is not
-    unqualified: a short query still scores low against a long memory
-    (whenever `|Q| < threshold * |M|` the memory is dropped however
-    relevant it is), and where `Q ⊆ M` the metric degenerates to exactly
-    Jaccard. So `|M|` is a trade against `min(|Q|, |M|)`, not a strict
-    improvement over it; the paragraph below is the argument for which
-    side of that trade is safer here.
-
-    Why ``|M|`` and not ``min(|Q|, |M|)`` (the textbook overlap coefficient):
-    the two agree only where the memory is the shorter side, and that is NOT
-    a safe assumption here. Replaying real production queries against the
-    live engine, `|Q| < |M|` held for 87 of 202 candidate pairs — 43%.
-    Memories are routinely LONGER than the prompt that retrieves them.
-    Wherever that happens `min()` selects the QUERY and the metric silently
-    becomes the converse measure: what fraction of the *prompt* appears in
-    the memory. That regime flip re-introduces exactly the query-length
-    dependence #3541 is about — a one-word prompt scores 1.0 against any
-    memory containing that word. Dividing by `|M|` unconditionally has no
-    such discontinuity.
-
-    Safety, relative to what production runs today: for any Q and M,
-
-        |Q ∩ M| / |Q ∪ M|  <=  |Q ∩ M| / |M|  <=  |Q ∩ M| / min(|Q|, |M|)
-
-    because `|Q ∪ M| >= |M| >= min(|Q|, |M|)`. So at a fixed threshold this
-    metric admits a SUPERSET of what the deployed Jaccard gate admits: no
-    memory that survives the gate today can be dropped by this change.
-    """
-    a = _overlap_tokens(query)
-    b = _overlap_tokens(memory_text)
-    if not a or not b:
-        return 0.0
-    return len(a & b) / len(b)
-
-
-def _filter_by_overlap(results, query: str, threshold: float):
-    """Drop memories whose containment overlap with the query is below the
-    threshold. Threshold <= 0 short-circuits to passthrough (no
-    iteration cost).
-
-    Returns (kept_results, dropped_count).
-    """
-    if threshold <= 0:
-        return results, 0
-    kept = []
-    dropped = 0
-    for m in results:
-        text = m.get("text", "") if isinstance(m, dict) else ""
-        if containment_overlap(query, text) >= threshold:
-            kept.append(m)
-        else:
-            dropped += 1
-    return kept, dropped
-
-
 def _result_final_score(m) -> float:
     """Return a result's engine relevance score (`scores.final`).
 
@@ -673,12 +517,11 @@ def _injected_score_stats(results) -> dict:
     """Relevance-score aggregates for the INJECTED set (post-head-slice).
 
     Switchroom #3541 review finding — recall-quality telemetry.
-    `recall_log.jsonl` records volume and plumbing only (`overlap_dropped`,
-    `capped`, `pre_cap_count`, `memory_ids`, `deadline_hit`). With the
-    overlap gate deliberately near-passthrough at 0.10, 100% of precision now
-    rests on the engine's `scores.final` plus the `recallMaxMemories`
-    head-slice — and no field observes that. Post-rollout, `overlap_dropped`
-    collapsing to 0 while `result_count` rises to the cap reads as
+    `recall_log.jsonl` records volume and plumbing only (`capped`,
+    `pre_cap_count`, `memory_ids`, `deadline_hit`). With the lexical overlap
+    gate removed (#3761) 100% of precision rests on the engine's
+    `scores.final` plus the `recallMaxMemories` head-slice — and no volume
+    field observes that. `result_count` rising to the cap reads as
     unambiguous success on every existing dashboard whether the reranker is
     good OR whether every agent is being fed 8 mediocre memories per turn.
     These three fields are what distinguishes those two worlds.
@@ -753,6 +596,174 @@ def _sort_by_final_score(results):
     return results
 
 
+# Key stamped onto every merged result naming the bank it came from. Private to
+# recall.py (leading underscore) and never rendered: `format_memories`
+# (lib/content.py:294) reads only `text` / `type` / `mentioned_at`, so an extra
+# key cannot leak into the injected `<hindsight_memories>` block.
+SOURCE_BANK_KEY = "_source_bank"
+
+
+def _tag_source_bank(bank_results, bank_id):
+    """Stamp `SOURCE_BANK_KEY` onto each result so slot reservation can tell
+    own-bank memories from additional-bank (profile / shared / sender) ones
+    after the global relevance sort has interleaved them.
+
+    Returns the same list. Non-dict entries are skipped rather than raising:
+    a malformed engine response must not take recall down.
+    """
+    for m in bank_results:
+        if isinstance(m, dict):
+            m[SOURCE_BANK_KEY] = bank_id
+    return bank_results
+
+
+def _reservable_slots(cap):
+    """How many of `cap` slots the per-bank floors may claim between them.
+
+    HALF the cap, rounded down. The other half is always awarded on pure global
+    relevance, and that headroom is what keeps these FLOORS rather than a fixed
+    quota. Without it the mechanism silently inverts at small caps: this fleet
+    runs `defaults.memory.recall.max_memories: 6` (switchroom.yaml — it cascades
+    to HINDSIGHT_RECALL_MAX_MEMORIES, which wins over the 8 stamped into the
+    plugin's settings.json), so floors of 4+2 would consume the entire cap,
+    `scores.final` would have no influence on composition on any turn where both
+    banks return, and the injected set would be a constant 4/2 split regardless
+    of relevance.
+
+    Scaling with the cap rather than clamping against a hardcoded 6 means the
+    invariant holds for any operator cap: at 6 the floors may claim 3, at 12 six,
+    at 4 two, at 1 none (reservation is simply off below cap 2).
+    """
+    if not isinstance(cap, int) or cap <= 0:
+        return 0
+    return cap // 2
+
+
+def _reserve_bank_slots(results, cap, own_bank_id, own_floor, additional_floor):
+    """Head-slice `results` to `cap`, guaranteeing a minimum number of slots to
+    the agent's OWN bank and to the additional (profile / shared / sender) banks.
+
+    Switchroom — profile-bank crowd-out fix. `_sort_by_final_score` above sorts
+    the merged multi-bank set by `scores.final` and the caller then head-slices
+    at `recallMaxMemories`. On a turn where BOTH banks return more candidates
+    than the cap, that head-slice is winner-take-all across banks: nothing stops
+    one bank's score distribution from filling every slot, and the shared
+    `ken-profile` bank's facts routinely outscore the agent's own working memory
+    because a profile bank is dense with short, highly-rerankable statements.
+    The result is an agent handed a dossier about its operator and none of its
+    own session memory.
+
+    Scope, precisely — this fixes score-based crowd-out among results that DID
+    return. It does NOT fix the own-bank timeout: a timed-out bank contributes
+    zero candidates, so `own_take` is 0 and reservation is a strict no-op there.
+    Measured across 1,036 multi-bank non-cache recalls on this host since
+    2026-07-20: own-dead/additional-alive is 67.0% of turns (83.5% for
+    overlord), and both-alive — the only state where reservation can act at all
+    — is 15.1% fleet-wide, 6.4% for overlord (6.8% once you also require more
+    candidates than the cap). The own-bank timeout is a separate, larger defect;
+    the `injected_own_bank_count` telemetry added alongside this is what makes
+    it legible per-turn.
+
+    Floors, not quotas, and enforced as such. Each side is guaranteed AT MOST
+    `floor` slots, only if it actually has that many results, and only up to
+    `_reservable_slots(cap)` (half the cap) between them; every other slot is
+    filled from the remaining candidates in pure global-relevance order. So at
+    the fleet's deployed cap of 6 with the shipped floors 2/1:
+
+      * own returns 10, profile returns 10  -> 2 own + 4 profile (profile-favoured
+                                              scores; own is never zeroed)
+      * own returns 10, profile returns 10  -> 5 own + 1 profile (own-favoured
+                                              scores; profile is never zeroed)
+      * own returns 0,  profile returns 10  -> 6 profile (no wasted slots)
+      * own returns 10, profile returns 0   -> 6 own      (no wasted slots)
+      * own returns 1,  profile returns 10  -> 1 own + 5 profile
+
+    Composition still moves with `scores.final` in both directions — that is the
+    property the half-cap headroom buys and the property a quota would destroy.
+
+    A floor of 0 disables reservation for that side. `cap <= 0` disables the cap
+    entirely (upstream contract) and is a passthrough here. When the floors sum
+    above the reservable half the OWN floor is honoured first — the crowd-out
+    being fixed is one-directional, and the agent's own memory is the side that
+    loses today.
+
+    Selection is stable and the returned list is re-sorted by relevance, so
+    reservation changes WHICH memories are injected, never the order they are
+    presented in. Returns `(selected, reserved_own, reserved_additional)` where
+    the two counts are the slots that would NOT have been won on global score
+    alone — i.e. the observable effect of this function, logged as telemetry.
+    """
+    if not isinstance(cap, int) or cap <= 0 or len(results) <= cap:
+        return results, 0, 0
+
+    baseline_ids = {id(m) for m in results[:cap]}
+
+    own = [m for m in results if _source_bank_of(m) == own_bank_id]
+    additional = [m for m in results if _source_bank_of(m) != own_bank_id]
+
+    # Floors compete for the reservable half only — never for the whole cap.
+    reservable = _reservable_slots(cap)
+    own_take = max(0, min(int(own_floor or 0), len(own), reservable))
+    additional_take = max(
+        0, min(int(additional_floor or 0), len(additional), reservable - own_take)
+    )
+
+    reserved = own[:own_take] + additional[:additional_take]
+    reserved_ids = {id(m) for m in reserved}
+
+    remaining = cap - len(reserved)
+    if remaining > 0:
+        for m in results:
+            if id(m) in reserved_ids:
+                continue
+            reserved.append(m)
+            reserved_ids.add(id(m))
+            remaining -= 1
+            if remaining == 0:
+                break
+
+    _sort_by_final_score(reserved)
+
+    promoted_own = sum(
+        1 for m in reserved if id(m) not in baseline_ids and _source_bank_of(m) == own_bank_id
+    )
+    promoted_additional = sum(
+        1 for m in reserved if id(m) not in baseline_ids and _source_bank_of(m) != own_bank_id
+    )
+    return reserved, promoted_own, promoted_additional
+
+
+def _injected_bank_composition(results, own_bank_id) -> dict:
+    """Own-bank / additional-bank split of the INJECTED set (post-head-slice).
+
+    Returns ``{"injected_own_bank_count", "injected_additional_bank_count"}``.
+    The two always sum to ``result_count``; results with no stamped source bank
+    (only possible from a cached row or a malformed engine response) count as
+    additional, so the own-bank number is never optimistic.
+    """
+    try:
+        own = sum(1 for m in results or [] if _source_bank_of(m) == own_bank_id)
+        return {
+            "injected_own_bank_count": own,
+            "injected_additional_bank_count": len(results or []) - own,
+        }
+    except Exception:
+        # Telemetry must never take recall down.
+        return {
+            "injected_own_bank_count": None,
+            "injected_additional_bank_count": None,
+        }
+
+
+def _source_bank_of(m):
+    """Read a result's stamped source bank, or None when absent/malformed."""
+    if isinstance(m, dict):
+        v = m.get(SOURCE_BANK_KEY)
+        if isinstance(v, str):
+            return v
+    return None
+
+
 def _is_timeout_error(exc: BaseException) -> bool:
     """True if `exc` is (or wraps) a network read/connect timeout.
 
@@ -1003,6 +1014,42 @@ _FALLBACK_TELEMETRY_ZERO = {
     "truncated": False,
 }
 
+# The UserPromptSubmit timeout Claude Code enforces on THIS script, mirrored
+# from `hooks/hooks.json` (which ships in this same package, so the two cannot
+# be configured apart by an operator). `test_hook_ceiling_matches_hooks_json`
+# fails if they ever drift.
+#
+# Overrun is not a degraded recall — Claude Code kills the hook and the turn
+# loses memories, the fallback AND the directives, which is strictly worse than
+# the bug this PR fixes. So every optional tail-end spend is budgeted against
+# what is actually left of the ceiling rather than against a flat constant.
+HOOK_CEILING_SECONDS = 12.0
+# Held back for work this arithmetic cannot see: CPython interpreter boot
+# before our first statement runs, plus rendering the additionalContext
+# payload, the recall_log write and teardown after the last budgeted step.
+HOOK_TAIL_RESERVE_SECONDS = 0.75
+# Anchor for the budget arithmetic. Set at the top of main() rather than at
+# import, so a process that invokes the hook more than once (the test suite,
+# and any future in-process driver) budgets each invocation independently
+# instead of inheriting the whole process lifetime. Import cost is charged
+# explicitly via `_IMPORT_ELAPSED_SECONDS` so re-anchoring loses nothing.
+_hook_start_monotonic = None
+
+
+def _begin_hook_budget():
+    """(Re-)anchor the hook budget clock. Call once at the top of main()."""
+    global _hook_start_monotonic
+    _hook_start_monotonic = time.monotonic() - _IMPORT_ELAPSED_SECONDS
+
+
+def _remaining_hook_budget_seconds():
+    """Seconds left before this hook risks breaching its UserPromptSubmit ceiling."""
+    if _hook_start_monotonic is None:  # pragma: no cover - defensive only
+        return HOOK_CEILING_SECONDS - HOOK_TAIL_RESERVE_SECONDS
+    elapsed = time.monotonic() - _hook_start_monotonic
+    return HOOK_CEILING_SECONDS - HOOK_TAIL_RESERVE_SECONDS - elapsed
+
+
 _FALLBACK_BLOCK_PREAMBLE = (
     "No stored memories matched this query — the fact layer may not have "
     "reconciled this session yet (e.g. an abrupt session death before boot "
@@ -1012,7 +1059,7 @@ _FALLBACK_BLOCK_PREAMBLE = (
 )
 
 
-def _transcript_grep_fallback(transcript_path, query, config):
+def _transcript_grep_fallback(transcript_path, query, config, budget_ms=None):
     """Bounded transcript-grep fallback for the empty-fact-layer window (#3369).
 
     Reads the CURRENT session transcript's tail (bounded bytes), keeps the most
@@ -1022,9 +1069,22 @@ def _transcript_grep_fallback(transcript_path, query, config):
     (``recallTranscriptFallbackMaxBytes``), matched turns
     (``recallTranscriptFallbackMaxTurns``), emitted characters
     (``recallTranscriptFallbackMaxChars``), and grep wall-time
-    (``recallTranscriptFallbackDeadlineMs``). The caller only invokes this when
-    all banks returned zero AND no slot hit its deadline, so a timed-out bank can
-    never masquerade as a genuinely empty fact layer.
+    (``recallTranscriptFallbackDeadlineMs``, default 1500). The caller invokes
+    this when all banks returned zero and none ERRORED — a hard bank error can
+    masquerade as a genuinely empty fact layer, so it still suppresses the
+    fallback. A per-bank TIMEOUT no longer does (#3757): timing out was the
+    common case, and suppressing on it left the agent with neither memories nor
+    fallback.
+
+    That inversion removed the gate that INCIDENTALLY protected the hook
+    ceiling, so the grep wall-time bound is now
+    ``min(recallTranscriptFallbackDeadlineMs, remaining hook budget)`` rather
+    than a flat 1.5s (#3760 review, Major 4). A flat 1.5s after a fully-elapsed
+    10s recall left ~0.5s for interpreter startup, config load, the transcript
+    read and output formatting — and an overrun costs the turn its directives
+    too. ``budget_ms`` is that remaining-budget clamp, supplied by the caller;
+    ``<= 0`` means "no time left", and the fallback declines rather than
+    gambling the hook.
 
     Returns ``(block_or_None, telemetry)``. Failure-safe: any error path returns
     ``(None, zeroed-telemetry)`` so the fallback can never break recall.
@@ -1046,6 +1106,12 @@ def _transcript_grep_fallback(transcript_path, query, config):
     max_turns = _int_cfg("recallTranscriptFallbackMaxTurns", 6)
     max_chars = _int_cfg("recallTranscriptFallbackMaxChars", 2000)
     deadline_ms = _int_cfg("recallTranscriptFallbackDeadlineMs", 1500)
+    if budget_ms is not None:
+        # Never spend more than the hook has left, even when the configured
+        # bound is larger. A caller with no budget left gets nothing at all.
+        deadline_ms = min(deadline_ms, int(budget_ms))
+        if deadline_ms <= 0:
+            return None, telemetry
 
     if max_turns <= 0 or max_chars <= 0 or max_bytes <= 0:
         return None, telemetry
@@ -1296,6 +1362,63 @@ def _combine_context(base, nudge) -> str:
     return "\n\n".join(parts)
 
 
+# Switchroom structural-fix #7 — WHAT failed, not merely THAT something did.
+#
+# Before this, every failure channel on a recall_log row was a BOOLEAN:
+# `timed_out`, `errored`, `deadline_hit`. So a log full of `errored: true`
+# could not answer the first question anyone asks during an incident — is this
+# a connection refused, a 500, a bad bank id, or an auth failure? Each implies
+# a different fix, and the log distinguished none of them. Worse, `recall.py`
+# exits 0 and Claude Code swallows hook stderr, so the exception text printed
+# on the `[Hindsight]` line reaches nobody: the JSONL row is the ONLY place
+# this information can survive.
+ERROR_TEXT_MAX_CHARS = 300
+
+
+def error_text(err) -> str | None:
+    """One-line, bounded, type-prefixed rendering of a failure. None when clean.
+
+    Type-prefixed because the message alone is often uselessly generic
+    (`''`, `'timed out'`); the exception class is frequently the most
+    diagnostic part. Bounded because rows are size-trimmed by line count and
+    an unbounded traceback-shaped message would evict real history.
+    """
+    if err is None:
+        return None
+    if isinstance(err, BaseException):
+        text = f"{type(err).__name__}: {err}"
+    else:
+        text = str(err)
+    text = " ".join(text.split())
+    if not text:
+        return None
+    return text[:ERROR_TEXT_MAX_CHARS]
+
+
+def recall_error_summary(bank_id, bank_timings, directives_timed_out=None) -> str | None:
+    """One row-level string answering "what actually failed on this turn?".
+
+    The per-bank `error` strings are the ground truth, but a row-level field is
+    what makes the log GREPPABLE: an operator triaging an incident wants
+    `jq -r .error recall_log.jsonl | sort | uniq -c`, not a nested walk.
+
+    Own bank first, because a side-bank failure does not mean the agent lost
+    its memory (same rationale as `degraded_recall_notice`). Side banks are
+    reported only when the own bank was fine, and are prefixed with their bank
+    id so the summary is never ambiguous about whose failure it names.
+    """
+    entries = [bt for bt in (bank_timings or []) if isinstance(bt, dict)]
+    own = next((bt for bt in entries if bt.get("bank_id") == bank_id), None) if bank_id else None
+    if own and own.get("error"):
+        return error_text(own["error"])
+    for bt in entries:
+        if bt.get("bank_id") != bank_id and bt.get("error"):
+            return error_text(f"additional bank '{bt.get('bank_id')}': {bt['error']}")
+    if directives_timed_out:
+        return "directives fetch timed out"
+    return None
+
+
 def degraded_recall_notice(bank_id, bank_timings) -> str:
     """Switchroom #3619 — return the degraded-recall disclosure for this turn,
     or "" when the agent's own bank answered.
@@ -1347,6 +1470,7 @@ def degraded_recall_notice(bank_id, bank_timings) -> str:
 
 
 def main():
+    _begin_hook_budget()
     config = load_config()
 
     if not config.get("autoRecall"):
@@ -1559,6 +1683,13 @@ def main():
                 "injected_score_min": None,
                 "injected_score_median": None,
                 "injected_score_max": None,
+                # Bank-composition telemetry — present for a uniformly queryable
+                # schema. Same reason as the score fields above: a cache hit
+                # replays a formatted block, not a per-bank result set.
+                "injected_own_bank_count": None,
+                "injected_additional_bank_count": None,
+                "reserved_own_slots": None,
+                "reserved_additional_slots": None,
                 "cache_hit": True,
                 # A3 stage-1 telemetry keys kept present for a uniformly
                 # queryable schema; a cache hit issues no bank HTTP, so there
@@ -1600,6 +1731,11 @@ def main():
                 # miscounted as an observed no-error recall — matching the
                 # deadline_hit / directives_timed_out convention above.
                 "bank_errored": None,
+                # Switchroom structural-fix #7 — WHAT failed, in words. None
+                # here (not "") for the same reason as the fields above: no
+                # banks ran, so this row observed no failure and must not be
+                # counted as one.
+                "error": None,
             })
             return
         debug_log(config, f"Recall cache MISS (key={cache_key[:12]}…)")
@@ -1646,7 +1782,35 @@ def main():
     if len(query) > recall_max_query_chars:
         query = query[:recall_max_query_chars]
 
-    debug_log(config, f"Recalling from bank '{bank_id}', query length: {len(query)}")
+    # Switchroom recall-latency fix (#3757) — bound the BM25 term count of the
+    # query we put on the wire. `recallMaxQueryChars` bounds CHARACTERS, which
+    # is not the cost driver: Hindsight OR-joins every token into one tsquery
+    # and Postgres native FTS ranks the whole matched set before the top-60
+    # heapsort, so cost tracks the number of DISTINCT TERMS. An 800-char
+    # composed query is ~96 distinct terms and matched 119,510 rows on the
+    # live `overlord` bank (14.0s for the 3-arm UNION, and up to 94s under
+    # load) — past the 8s client timeout, which is why 96.8% of that agent's
+    # own-bank recalls returned nothing. Shaped to 24 terms the same query
+    # matches 48,433 rows in 2.5-2.8s.
+    #
+    # `search_query` is what the SERVER sees. `query` (unshaped) stays the
+    # client-side lexical reference: the `recallMinOverlap` containment gate
+    # and the transcript-grep fallback both measure against the user's real
+    # words, so shaping cannot silently move their thresholds. `query_chars`
+    # telemetry also stays on the unshaped value for continuity with the
+    # existing recall_log history.
+    search_query = shape_recall_query(
+        query,
+        recall_query_text,
+        max_tokens=config.get("recallQueryMaxTokens", 24),
+        stop_terms=config.get("recallQueryStopTerms") or (),
+    )
+
+    debug_log(
+        config,
+        f"Recalling from bank '{bank_id}', query length: {len(query)}, "
+        f"search terms: {len(set(tokenize_for_bm25(search_query)))}",
+    )
 
     # Fetch active directives FIRST (independent of recall — even if recall
     # finds no memories, an agent with active directives still needs them
@@ -1680,11 +1844,18 @@ def main():
             ttl_seconds=config.get("directivesCacheTtlSeconds", DIRECTIVES_CACHE_TTL_SECONDS),
         )
 
-    def _make_bank_task(target_bank_id, b_tags, b_tags_match, b_tag_groups):
+    try:
+        recall_request_timeout = float(config.get("recallRequestTimeoutSeconds", 12))
+    except (TypeError, ValueError):
+        recall_request_timeout = 12.0
+    if recall_request_timeout <= 0:
+        recall_request_timeout = 12.0
+
+    def _make_bank_task(target_bank_id, b_tags, b_tags_match, b_tag_groups, timeout_override=None):
         def _bank_task():
             return client.recall(
                 bank_id=target_bank_id,
-                query=query,
+                query=search_query,
                 max_tokens=config.get("recallMaxTokens", 1024),
                 budget=config.get("recallBudget", "mid"),
                 types=config.get("recallTypes"),
@@ -1698,12 +1869,31 @@ def main():
                 # slots for denser coverage inside the same budget. On by default;
                 # operators can pin off via `recallPreferObservations: false`.
                 prefer_observations=config.get("recallPreferObservations", True),
-                # 8s in-script per-request timeout: even parallelised, each bank
+                # Per-request in-script timeout: even parallelised, each bank
                 # carries its own hard deadline so a single hung bank returns
                 # cleanly with no memories rather than sitting on the shared
-                # deadline. Tightened from 10s in v0.13.22 (2026-05-24 breach
-                # audit); the shared deadline below is the outer ceiling guard.
-                timeout=8,
+                # deadline. Tightened from 10s to 8s in v0.13.22 (2026-05-24
+                # breach audit); the shared deadline below is the outer ceiling
+                # guard.
+                #
+                # Switchroom #3757: no longer a hardcoded literal. It was the
+                # binding constraint on a slow bank (96.8% of overlord's
+                # own-bank recalls hit exactly 8s and returned NOTHING), and a
+                # hand-patch of the installed copy does not survive
+                # `switchroom apply` — the plugin dir is re-copied from
+                # `vendor/hindsight-memory` on every reconcile. Default raised
+                # to 12s, matching the UserPromptSubmit hook's own 12s budget
+                # (hooks/hooks.json). Note the SHARED multi-bank deadline
+                # (`recallParallelDeadlineSeconds`, default 10s) is the tighter
+                # outer bound in the default configuration, so at the default
+                # this timeout only binds when an operator raises that. SAFETY
+                # NET behind the query-shaping fix above, not the fix.
+                # Operator knob: `memory.recall.request_timeout_seconds`.
+                timeout=(
+                    recall_request_timeout
+                    if timeout_override is None
+                    else timeout_override
+                ),
             )
         return _bank_task
 
@@ -1796,7 +1986,7 @@ def main():
                 )
                 if bank_results:
                     debug_log(config, f"Got {len(bank_results)} memories from bank '{b_id}'")
-                    results = results + bank_results
+                    results = results + _tag_source_bank(bank_results, b_id)
             elif b_outcome.error is not None:
                 # Own bank failure surfaces on stderr (journald signal); extra
                 # banks are debug-only, matching the pre-A3 serial behaviour.
@@ -1811,6 +2001,12 @@ def main():
                 "elapsed_ms": b_outcome.elapsed_ms if b_outcome.elapsed_ms is not None else 0,
                 "timed_out": b_timed_out,
                 "errored": b_errored,
+                # The failure TEXT, not just the flags. On the parallel path a
+                # slot abandoned at the shared deadline carries no exception at
+                # all, so name that case explicitly rather than logging null
+                # and leaving "why is this row empty" unanswerable.
+                "error": error_text(b_outcome.error)
+                or (None if b_outcome.completed else "abandoned at the shared recall deadline"),
             })
     else:
         # Pre-A3 serial path (rollback lever, HINDSIGHT_RECALL_PARALLEL=false).
@@ -1820,6 +2016,14 @@ def main():
         # own-bank debug line, logs the directives block after the bank loop
         # rather than before, and __main__ still os._exit(0)s on completion.
         recall_mode = "serial"
+        # #3760 review, Major 4. This path has no outer deadline — bank
+        # latencies SUM — so raising the per-bank timeout 8s -> 12s would let
+        # two banks spend 24s against a 12s hook ceiling. Each bank is instead
+        # clamped to whatever the hook has left when its turn comes, so the
+        # serial path can no longer breach the ceiling no matter how many banks
+        # are configured. `deadline_budget_ms` stays None on the log row: this
+        # is a per-bank clamp, not the parallel path's shared deadline, and
+        # conflating them would corrupt the serial-vs-parallel comparison.
         deadline_budget_ms = None
         deadline_effective_ms = None
         _directives_start = time.monotonic()
@@ -1834,16 +2038,29 @@ def main():
             _bank_start = time.monotonic()
             _bank_timed_out = False
             _bank_errored = False
+            _bank_error = None
             try:
-                response = _make_bank_task(b_id, b_tags, b_tags_match, b_tag_groups)()
+                _bank_budget = _remaining_hook_budget_seconds()
+                if _bank_budget <= 0:
+                    raise TimeoutError(
+                        f"hook budget exhausted before bank '{b_id}' was queried"
+                    )
+                response = _make_bank_task(
+                    b_id,
+                    b_tags,
+                    b_tags_match,
+                    b_tag_groups,
+                    timeout_override=min(recall_request_timeout, _bank_budget),
+                )()
                 bank_results = response.get("results", []) if isinstance(response, dict) else []
                 if bank_results:
                     debug_log(config, f"Got {len(bank_results)} memories from bank '{b_id}'")
-                    results = results + bank_results
+                    results = results + _tag_source_bank(bank_results, b_id)
             except Exception as e:
                 _bank_timed_out = _is_timeout_error(e)
                 # Non-timeout error → hard outage (see parallel-path note above).
                 _bank_errored = not _bank_timed_out
+                _bank_error = error_text(e)
                 if b_id == bank_id:
                     print(f"[Hindsight] Recall failed: {e}", file=sys.stderr)
                 else:
@@ -1853,6 +2070,7 @@ def main():
                 "elapsed_ms": int((time.monotonic() - _bank_start) * 1000),
                 "timed_out": _bank_timed_out,
                 "errored": _bank_errored,
+                "error": _bank_error,
             })
 
     # Switchroom hindsight-leverage A3 — FINALIZED `deadline_hit`: True when ANY
@@ -1908,28 +2126,6 @@ def main():
                 f"memories (active_thread_id={active_thread_id})",
             )
 
-    # Switchroom #475 — lexical-overlap relevance gate. Drops memories
-    # whose containment overlap with the query is below
-    # `recallMinOverlap` (default 0.0 = disabled). Runs after the
-    # demote filter so the threshold sees the operator-curated set.
-    # #3541: the metric is containment, NOT Jaccard — see the design
-    # note above `_OVERLAP_STOPWORDS`. Jaccard made the gate a function
-    # of prompt length and it discarded 97.1% of reranked candidates.
-    overlap_threshold = config.get("recallMinOverlap", 0.0)
-    if isinstance(overlap_threshold, (int, float)) and overlap_threshold > 0:
-        pre_overlap_count = len(results)
-        results, overlap_dropped = _filter_by_overlap(
-            results, query, float(overlap_threshold)
-        )
-        if overlap_dropped > 0:
-            debug_log(
-                config,
-                f"Overlap gate dropped {overlap_dropped}/{pre_overlap_count} "
-                f"memories below threshold {overlap_threshold}",
-            )
-    else:
-        overlap_dropped = 0
-
     # Switchroom hindsight-leverage PR5 — per-tag score penalty. Applied
     # IMMEDIATELY before the relevance sort so a down-weighted tag (e.g.
     # `sidechain: 0.8`) reorders the merged set without dropping anything. This
@@ -1959,6 +2155,8 @@ def main():
     recall_max_memories = config.get("recallMaxMemories", 0)
     pre_cap_count = len(results)
     capped = False
+    reserved_own = 0
+    reserved_additional = 0
     if (
         isinstance(recall_max_memories, int)
         and recall_max_memories > 0
@@ -1969,7 +2167,29 @@ def main():
             f"Capping {len(results)} memories to {recall_max_memories} "
             f"(set HINDSIGHT_RECALL_MAX_MEMORIES=0 to disable)",
         )
-        results = results[:recall_max_memories]
+        # Switchroom — per-bank slot reservation. Head-slicing a globally sorted
+        # merged set is winner-take-all across banks: when both banks return more
+        # candidates than the cap, one bank's score distribution can fill every
+        # slot and the agent's own working memory is crowded out entirely.
+        # Guarantee a floor to each side (bounded to half the cap) before the
+        # remaining slots go to pure global relevance. This addresses score-based
+        # crowd-out only — a timed-out bank contributes no candidates and
+        # reservation is a no-op there; see _reserve_bank_slots for the measured
+        # share of turns it can bind on. Floors of 0/0 restore the head-slice.
+        results, reserved_own, reserved_additional = _reserve_bank_slots(
+            results,
+            recall_max_memories,
+            bank_id,
+            config.get("recallOwnBankMinSlots", 0),
+            config.get("recallAdditionalBankMinSlots", 0),
+        )
+        if reserved_own or reserved_additional:
+            debug_log(
+                config,
+                f"Bank slot reservation promoted {reserved_own} own-bank / "
+                f"{reserved_additional} additional-bank memories over "
+                f"higher-scoring ones",
+            )
         capped = True
 
     memories_block = None
@@ -2023,18 +2243,29 @@ def main():
     # On by default; HINDSIGHT_RECALL_TRANSCRIPT_FALLBACK=false is the rollback
     # lever. Mutually exclusive with memories_block by construction: a non-empty
     # memories_block requires results, which requires pre_filter_count > 0.
+    #
+    # Switchroom #3757 — `deadline_hit` NO LONGER SUPPRESSES the fallback.
+    # The original gate was written when a deadline meant "the fact layer is
+    # unknown, don't guess". In practice a timeout was the COMMON case (96.8%
+    # of overlord's own-bank recalls over the 7 days to 2026-07-27), and the
+    # gate meant a timed-out turn got neither memories NOR the fallback — the
+    # agent went in blind. A timeout and an outage are different: on a timeout
+    # the banks are healthy and reachable, we simply ran out of time, so the
+    # bounded transcript grep is strictly better than nothing. A hard bank
+    # ERROR still suppresses it (`bank_errored`), because that genuinely can
+    # masquerade as an empty fact layer while the store is down.
     transcript_fallback_block = None
     transcript_fallback_telemetry = dict(_FALLBACK_TELEMETRY_ZERO)
     if (
         config.get("recallTranscriptFallback", True)
         and pre_filter_count == 0
-        and not deadline_hit
         and not bank_errored
     ):
         transcript_fallback_block, transcript_fallback_telemetry = _transcript_grep_fallback(
             hook_input.get("transcript_path", ""),
             query,
             config,
+            budget_ms=_remaining_hook_budget_seconds() * 1000.0,
         )
         if transcript_fallback_block:
             debug_log(
@@ -2102,7 +2333,6 @@ def main():
         # doctor's directive-count check will be FAILing too.
         "directives_omitted": count_omitted_directives(directives),
         "demoted_count": demoted_count,
-        "overlap_dropped": overlap_dropped,
         "capped": capped,
         "pre_cap_count": pre_cap_count,
         "memory_ids": [
@@ -2115,6 +2345,19 @@ def main():
         # alone can't distinguish "reranker is working" from "8 mediocre
         # memories per turn" now that the overlap gate is near-passthrough.
         **_injected_score_stats(results),
+        # Switchroom — injected BANK COMPOSITION. `result_count` above is a
+        # volume signal and cannot distinguish "6 own-bank memories" from "6
+        # profile-bank memories because the own bank timed out", which is
+        # exactly how the own-bank timeout outage stayed invisible for weeks:
+        # a fully-timed-out own bank still logged result_count == cap. These
+        # two counts always sum to `result_count`, so an own-bank collapse is
+        # readable off the log row without joining `bank_timings`.
+        **_injected_bank_composition(results, bank_id),
+        # Slots the reservation floors handed to a side that would have lost
+        # them on global relevance alone — the observable effect of
+        # `_reserve_bank_slots` (0/0 when the floors are off or non-binding).
+        "reserved_own_slots": reserved_own,
+        "reserved_additional_slots": reserved_additional,
         "cache_hit": False,
         # Switchroom A3 stage-1 telemetry (hindsight-leverage PR 1) — per-bank
         # latency + timeout breakdown, directives-fetch latency, total
@@ -2177,6 +2420,14 @@ def main():
         "transcript_fallback_bytes_read": transcript_fallback_telemetry["bytes_read"],
         "transcript_fallback_elapsed_ms": transcript_fallback_telemetry["elapsed_ms"],
         "transcript_fallback_truncated": transcript_fallback_telemetry["truncated"],
+        # Switchroom structural-fix #7 — WHAT failed, in words. Every other
+        # failure channel on this row is a BOOLEAN (`deadline_hit`,
+        # `bank_errored`, per-bank `timed_out`/`errored`), which says THAT
+        # something broke and never WHICH thing, so an incident could not be
+        # triaged from the log at all. recall.py exits 0 and Claude Code
+        # swallows hook stderr, so this row is the only place the reason can
+        # survive. None on a healthy turn.
+        "error": recall_error_summary(bank_id, bank_timings, directives_timed_out),
     })
 
     # Switchroom #3619 — DEGRADED-RECALL DISCLOSURE. See