switchroom 0.19.24 → 0.19.26

package/vendor/hindsight-memory/scripts/lib/config.py

@@ -29,21 +29,23 @@ DEFAULTS = {
     # formatting. Set to 0 (or any non-positive value) to disable the cap
     # and inject everything Hindsight returns.
     "recallMaxMemories": 12,
-    # Switchroom-local: minimum lexical (containment) overlap between the
-    # user's query terms and a memory's text terms. Memories below this
-    # threshold are dropped before formatting. 0.0 disables the gate
-    # (current behaviour: inject everything Hindsight returns up to the
-    # count cap). NOTE: Hindsight's HTTP recall API DOES return per-result
-    # relevance scores (`scores.final`, plus `.semantic`/`.keyword`/
-    # `.reranker`) — verified at runtime — and recall.py now reads and
-    # sorts the merged set by `scores.final`. This lexical gate is a
-    # separate quality filter layered on top — see #475. The metric is
-    # containment, `|Q n M| / |M|`, not Jaccard: dividing by the union made
-    # the score a function of prompt length rather than relevance — see
-    # #3541 and recall.py's design note. At the 0.10 fleet default this is
-    # close to a passthrough (a <=10-token memory clears it on one shared
-    # word); precision is the engine rerank's job, not this gate's.
-    "recallMinOverlap": 0.0,
+    # Switchroom-local: per-bank slot FLOORS inside `recallMaxMemories`. The
+    # merged multi-bank set is sorted globally by `scores.final` and then
+    # head-sliced, which is winner-take-all across banks: when both banks return
+    # more candidates than the cap, one bank's score distribution can fill every
+    # slot and the agent gets a dossier about its operator with none of its own
+    # working memory. These are FLOORS, not quotas: each side gets at most this
+    # many slots, only if it has that many results, and only up to HALF the cap
+    # between them — the other half is always awarded on pure global relevance,
+    # so composition still moves with the scores. 0 disables reservation for
+    # that side (the pure pre-fix head-slice). Env:
+    # HINDSIGHT_RECALL_OWN_BANK_MIN_SLOTS /
+    # HINDSIGHT_RECALL_ADDITIONAL_BANK_MIN_SLOTS. See recall.py's
+    # `_reserve_bank_slots` and `_reservable_slots`. Vendor default is 0/0
+    # (off); switchroom's scaffold opts in with 2 own / 1 additional against the
+    # cap of 6 its fleet actually deploys.
+    "recallOwnBankMinSlots": 0,
+    "recallAdditionalBankMinSlots": 0,
     "recallTypes": ["world", "experience"],
     # Switchroom-local: when True (default; Ken-approved ON) recall biases
     # toward synthesized `observation`-tier facts. Escape hatch: pin off via
@@ -91,6 +93,29 @@ DEFAULTS = {
     # by recallTranscriptTailBytes so the added per-turn read stays O(1).
     "recallContextTurns": 2,
     "recallMaxQueryChars": 800,
+    # Switchroom #3757 — BM25 term budget for the query put on the wire.
+    # `recallMaxQueryChars` bounds CHARACTERS; the server's keyword arm costs
+    # per DISTINCT TERM, because it OR-joins every token into one tsquery and
+    # Postgres native FTS ranks the entire matched set before the top-60
+    # heapsort. An 800-char composed query is ~96 distinct terms and matched
+    # 119,510 rows on the live `overlord` bank — 14.0s for the 3-arm UNION,
+    # and up to 94s under load, past the
+    # per-bank client timeout, so the agent got NOTHING on 96.8% of its
+    # own-bank recalls in the 7 days to 2026-07-27. 24 terms measures at
+    # 48,433 rows / 2.7s on the same bank while keeping the high-signal terms
+    # of the latest turn. Selection is recency-first (latest turn beats prior
+    # context), then by a selectivity proxy — see lib/content.shape_recall_query.
+    # 0 disables shaping entirely (rollback lever).
+    # Operator knob: `memory.recall.query_max_tokens` in switchroom.yaml.
+    "recallQueryMaxTokens": 24,
+    # Switchroom #3757 — extra terms to drop from the BM25 query on top of the
+    # built-in English stopword list. For BANK-SPECIFIC high-document-frequency
+    # words a generic stoplist cannot know about: on `overlord`, `switchroom`
+    # matches 27,090 of 135,443 rows (20%) and `agent` 26,496 (20%), purely
+    # because that is what the corpus is about. Empty by default — an operator
+    # sets it per-agent after reading `switchroom memory recall-log <agent>`.
+    # Operator knob: `memory.recall.query_stop_terms` in switchroom.yaml.
+    "recallQueryStopTerms": [],
     # Switchroom hindsight-leverage A2 (PR2) — latency bound for the multi-turn
     # composition. With recallContextTurns>1 now the default, EVERY recall reads
     # the transcript to slice the last N human turns. A long session's .jsonl can
@@ -204,6 +229,18 @@ DEFAULTS = {
     # can never push the hook past its ceiling. Slots still unfinished when the
     # deadline elapses are abandoned (daemon threads) and marked timed_out.
     "recallParallelDeadlineSeconds": 10,
+    # Switchroom #3757 — per-bank HTTP read timeout (seconds) for one recall
+    # request. Was a hardcoded `timeout=8` in recall.py, which made it BOTH the
+    # binding constraint on a slow bank AND un-tunable without hand-editing the
+    # installed plugin — and a hand-edit does not survive `switchroom apply`,
+    # which re-copies the plugin from `vendor/hindsight-memory` (that revert is
+    # exactly what put the 8s literal back on 2026-07-27). 12s matches the
+    # UserPromptSubmit hook ceiling in hooks.json; the shared
+    # `recallParallelDeadlineSeconds` (10s) is the tighter outer guard in the
+    # default configuration, so this is a per-request safety net rather than
+    # the primary bound. Non-positive values fall back to the default.
+    # Operator knob: `memory.recall.request_timeout_seconds` in switchroom.yaml.
+    "recallRequestTimeoutSeconds": 12,
     # Switchroom hindsight-leverage E1 / PR8 (#3369) — bounded transcript-grep
     # fallback. Boot reconciliation (reconcile_tail.py) closes the crash-loss
     # window at the NEXT SessionStart, but between an abrupt kill and that boot,
@@ -282,10 +319,11 @@ ENV_OVERRIDES = {
     # agents.<name>.memory.recall.max_memories (cascading through
     # defaults.memory.recall.max_memories) when present in switchroom.yaml.
     "HINDSIGHT_RECALL_MAX_MEMORIES": ("recallMaxMemories", int),
-    # Switchroom-local: lexical-overlap threshold (#475). Float in
-    # [0.0, 1.0]. Set by start.sh from agents.<name>.memory.recall.min_overlap
-    # (cascading through defaults). 0.0 = off (current behaviour).
-    "HINDSIGHT_RECALL_MIN_OVERLAP": ("recallMinOverlap", float),
+    # Switchroom-local: per-bank slot floors inside the count cap. Set by
+    # start.sh from agents.<name>.memory.recall.own_bank_min_slots /
+    # .additional_bank_min_slots (cascading through defaults). 0 = off.
+    "HINDSIGHT_RECALL_OWN_BANK_MIN_SLOTS": ("recallOwnBankMinSlots", int),
+    "HINDSIGHT_RECALL_ADDITIONAL_BANK_MIN_SLOTS": ("recallAdditionalBankMinSlots", int),
     # Switchroom-local: recall fact types (comma-separated). Set by start.sh
     # from agents.<name>.memory.recall.types only when the operator overrode
     # the switchroom default (world,experience,observation) — i.e. the
@@ -322,6 +360,10 @@ ENV_OVERRIDES = {
     "HINDSIGHT_RECALL_TRANSCRIPT_FALLBACK_MAX_CHARS": ("recallTranscriptFallbackMaxChars", int),
     "HINDSIGHT_RECALL_TRANSCRIPT_FALLBACK_DEADLINE_MS": ("recallTranscriptFallbackDeadlineMs", int),
     "HINDSIGHT_RECALL_MAX_QUERY_CHARS": ("recallMaxQueryChars", int),
+    # Switchroom #3757 — BM25 query shaping + per-request timeout.
+    "HINDSIGHT_RECALL_QUERY_MAX_TOKENS": ("recallQueryMaxTokens", int),
+    "HINDSIGHT_RECALL_QUERY_STOP_TERMS": ("recallQueryStopTerms", list),
+    "HINDSIGHT_RECALL_REQUEST_TIMEOUT_SECONDS": ("recallRequestTimeoutSeconds", int),
     "HINDSIGHT_RECALL_CONTEXT_TURNS": ("recallContextTurns", int),
     # Switchroom hindsight-leverage A2 — byte-tail bound for the multi-turn
     # transcript read (0 = read whole file / rollback lever).

package/vendor/hindsight-memory/scripts/lib/content.py

@@ -115,7 +115,18 @@ def compose_recall_query(
         if role == "user" and content == latest:
             continue
 
-        context_lines.append(f"{role}: {content}")
+        # Switchroom recall-latency fix (#3757): NO ``{role}: `` prefix.
+        # These labels were formatting scaffolding, never search terms, and
+        # they were the single most expensive thing this hook put on the wire.
+        # Hindsight's BM25 arm OR-joins every query token, so ``user`` and
+        # ``assistant`` widened the match set by the two highest-document-
+        # frequency terms in a mature bank (measured on bank ``overlord``,
+        # 135,443 rows: ``user`` = 67,363 rows = 50% of the bank,
+        # ``assistant`` = 29,942 = 22%). Dropping just these two labels cut a
+        # production-shaped 3-arm BM25 UNION from 119,510 ranked rows / 14.0s
+        # to 86,653 / 11.8s. Turn separation is preserved by the newline join,
+        # which is all the embedding arm needs.
+        context_lines.append(content)
 
     if not context_lines:
         return latest
@@ -181,6 +192,370 @@ def truncate_recall_query(query: str, latest_query: str, max_chars: int) -> str:
     return latest_only
 
 
+# ---------------------------------------------------------------------------
+# Recall: search-query shaping (BM25 token budget)
+# ---------------------------------------------------------------------------
+#
+# Switchroom recall-latency fix (#3757). Hindsight's keyword arm OR-joins
+# EVERY token of the query into one `to_tsquery` disjunction, and Postgres
+# native FTS cannot top-k from the GIN index — it computes `ts_rank_cd` on
+# every matching row before the top-60 heapsort. So BM25 cost grows with the
+# size of the matched set, which grows with query length. Measured on the live
+# `overlord` bank (135,565 memory_units, 3 fact-type arms, LIMIT 60 each,
+# production-shaped composed query):
+#
+#     as shipped (96 distinct terms)     119,510 rows ranked    14.0 s
+#     role labels + header stripped      86,653 rows ranked     11.8 s
+#     + capped to 24 BM25 terms          48,433 rows ranked      2.7 s
+#
+# The 8s client timeout then fired on 96.8% of overlord's own-bank recalls
+# over the 7 days to 2026-07-27, so the agent got zero memories.
+#
+# We therefore shape the query into a bounded set of the most selective terms
+# before it goes on the wire. Only the SERVER-BOUND query is shaped; the
+# client-side lexical surfaces (the `recallMinOverlap` containment gate and the
+# transcript-grep fallback) keep the unshaped text, so this change cannot move
+# their thresholds.
+
+# Common English function words.
+#
+# MEASURED CAVEAT, so nobody over-credits this list: on the CURRENT backend
+# (Postgres native tsvector with the `english` configuration) these are already
+# stopped by Postgres itself — `to_tsquery('english', 'the | user | worktree')`
+# returns `'user' | 'worktre'`. Removing them client-side moved the matched set
+# by only 86,653 → ~86,300 rows on the `overlord` bank. Their real job here is
+# BUDGET: every function word we send would otherwise consume one of the
+# `max_tokens` slots that a content word needs. They also matter directly on
+# the non-native backends Hindsight supports (vchord / pgroonga / pg_search /
+# pg_textsearch), none of which strip English stopwords for us.
+#
+# Deliberately conservative: only closed-class words (determiners, pronouns,
+# prepositions, auxiliaries, conjunctions), never content words.
+BM25_STOPWORDS = frozenset(
+    """
+    a about above after again against all am an and any are aren as at
+    be because been before being below between both but by
+    can cannot could couldn
+    did didn do does doesn doing don down during
+    each few for from further
+    had hadn has hasn have haven having he her here hers herself him himself his
+    how
+    i if in into is isn it its itself
+    just
+    ll
+    me more most mustn my myself
+    no nor not now
+    of off on once only or other ought our ours ourselves out over own
+    re
+    s same shan she should shouldn so some such
+    t than that the their theirs them themselves then there these they this
+    those through to too
+    under until up
+    ve very
+    was wasn we were weren what when where which while who whom why will with
+    won would wouldn
+    you your yours yourself yourselves
+    """.split()
+)
+
+# Role labels this hook used to prefix onto context turns, plus the composed
+# query's own section header. Structural scaffolding, never search terms.
+_ROLE_LABEL_RE = re.compile(r"(?mi)^\s*(?:user|assistant|system|tool)\s*:[ \t]*")
+_CONTEXT_HEADER_RE = re.compile(r"(?mi)^\s*prior context\s*:[ \t]*$")
+
+# Mirrors the server's own BM25 tokenizer
+# (hindsight_api/engine/search/retrieval.py::tokenize_query): lowercase, strip
+# punctuation, split on whitespace, then APPEND intact "compound" tokens —
+# word-char runs joined by . / - (semvers, paths, hyphenated identifiers) —
+# because the server emits those alongside their fragments. Keeping the two in
+# sync is what makes the cap mean "N terms in the tsquery", not "N words we
+# happened to send".
+_COMPOUND_TOKEN_RE = re.compile(r"\w+(?:[./-]\w+)+")
+
+
+def tokenize_for_bm25(text: str) -> list:
+    """Tokenize ``text`` the way Hindsight's BM25 arm will.
+
+    Port of ``tokenize_query`` in the Hindsight engine. Returns the token list
+    (with duplicates, in order); the tsquery ORs the distinct values.
+    """
+    lowered = text.lower()
+    tokens = re.sub(r"[^\w\s]", " ", lowered).split()
+    if not tokens:
+        return []
+    for match in _COMPOUND_TOKEN_RE.finditer(lowered):
+        compound = match.group(0)
+        if compound not in tokens:
+            tokens.append(compound)
+    return tokens
+
+
+def strip_query_scaffolding(text: str) -> str:
+    """Remove role labels and the "Prior context:" header from a query.
+
+    Defence in depth for #3757: ``compose_recall_query`` no longer emits the
+    ``user:`` / ``assistant:`` prefixes, but a transcript turn can legitimately
+    *contain* such a line, and older composed strings may still reach here.
+    """
+    text = _CONTEXT_HEADER_RE.sub("", text)
+    return _ROLE_LABEL_RE.sub("", text)
+
+
+_ENGLISH_WORDS_FILE = os.path.join(os.path.dirname(__file__), "english_words.txt")
+_ENGLISH_WORDS_CACHE = None
+
+
+def common_english_words() -> frozenset:
+    """Lazily load the common-English demotion list (see english_words.txt).
+
+    Read once per process and cached. A missing or unreadable file degrades to
+    an empty set — shaping still works, it just loses the demotion signal — so
+    a packaging slip can never break recall on the critical path.
+    """
+    global _ENGLISH_WORDS_CACHE
+    if _ENGLISH_WORDS_CACHE is None:
+        words = set()
+        try:
+            with open(_ENGLISH_WORDS_FILE, encoding="utf-8") as handle:
+                for line in handle:
+                    word = line.strip()
+                    if word and not word.startswith("#"):
+                        words.add(word)
+        except OSError:
+            words = set()
+        _ENGLISH_WORDS_CACHE = frozenset(words)
+    return _ENGLISH_WORDS_CACHE
+
+
+# How much each signal is worth. Digit and compound are equal because both are
+# measured to predict the same thing (an identifier); the English demotion is
+# deliberately smaller than either, so a digit-bearing English word still wins.
+_SCORE_DIGIT = 3.0
+_SCORE_COMPOUND = 3.0
+_SCORE_NOT_COMMON_ENGLISH = 2.0
+# Recency is a WEIGHT, not a tier (#3760 review, Blocker 2). Sized below the
+# shape signals on purpose: a distinctive prior-context identifier (`nginx`,
+# score 2.0) must outrank a generic latest-turn English word (score 0.0 + 1.5),
+# but between two terms of equal merit the one the user just typed wins.
+_SCORE_RECENCY = 1.5
+
+
+def _selectivity_score(token: str) -> float:
+    """Deterministic proxy for how discriminating ``token`` is.
+
+    True per-token document frequency is not available client-side: Hindsight
+    exposes no term-stats endpoint, and one ``count(*)`` probe per token would
+    cost more than the query it is trying to speed up. Every signal below was
+    checked against real df, measured with ``ts_stat`` over the live `overlord`
+    bank (135,565 units) and five other topically-unrelated banks:
+
+      * carries a digit (+3.0) — versions, issue numbers, ids, dates. The one
+        shape that reliably predicts a low df: median df 0.000, and only 6.8%
+        of digit-bearing tokens exceed 1% of the bank, against 33-50% for every
+        other shape class.
+      * is a compound token (+3.0) — ``v0.19.17``, ``src/agents/scaffold.ts``.
+      * is NOT a common English word (+2.0) — see ``english_words.txt``. df
+        cannot separate contentless English from domain identifiers (both are
+        rare: `particularly` 0.022%, `situation` 0.018%, vs `npm` 1.38%,
+        `worktree` 1.85%), but English-word membership can, and it is exactly
+        the separation the budget needs.
+
+    LENGTH IS DELIBERATELY ABSENT. The previous revision added
+    ``min(len(token), 12) / 4.0``. Measured median `overlord` df by token
+    length is flat-to-RISING — len2 0.0022, len3 0.0032, len4 0.0042,
+    len5 0.0070, len6 0.0103, len7 0.0102, len9 0.0068, len11 0.0055,
+    len13 0.0093 — so length predicts nothing in either direction. Inverting
+    it would have been as unjustified as the original; it is dropped instead.
+    Case was checked too and also rejected: `Capitalised` tokens have the
+    HIGHEST rate of df > 1% (50.6%), and ALLCAPS the highest mean df (0.034).
+
+    Known limit: bank-specific high-df content words (`agent` and `switchroom`
+    each match ~20% of the `overlord` bank) still score normally. They are not
+    a cost problem — ``max_tokens`` bounds tsquery cost regardless of which
+    terms are chosen — only a slot-allocation one, and they remain the job of
+    the operator-set ``recallQueryStopTerms``.
+    """
+    score = 0.0
+    if any(ch.isdigit() for ch in token):
+        score += _SCORE_DIGIT
+    if _COMPOUND_TOKEN_RE.fullmatch(token):
+        score += _SCORE_COMPOUND
+    if token not in common_english_words():
+        score += _SCORE_NOT_COMMON_ENGLISH
+    return score
+
+
+def shape_recall_query(
+    query: str,
+    latest_query: str = "",
+    max_tokens: int = 24,
+    stop_terms=None,
+) -> str:
+    """Bound the BM25 cost of ``query`` by capping its distinct tsquery terms.
+
+    Returns a space-joined term string in the query's original word order (so
+    the embedding arm still sees the text's natural sequence, just with the
+    scaffolding and function words removed).
+
+    Selection is RESERVE-then-FILL, in three parts:
+
+      1. ``max_tokens // 3`` slots go to the highest-:func:`_selectivity_score`
+         terms in the whole window, recency deliberately excluded from that
+         score. The most discriminating terms present survive regardless of
+         which turn they came from.
+      2. ``max_tokens // 3`` slots go to the best terms of the latest turn, so
+         a prior turn dense in high-merit tokens (a pasted stack trace, a list
+         of ids) cannot cost the user the question they just asked.
+      3. Everything left is filled by ``_selectivity_score`` plus
+         ``_SCORE_RECENCY`` for terms appearing in ``latest_query``, ties broken
+         by first appearance.
+
+    Both reserves are CEILINGS, not allocations: a side with fewer terms simply
+    leaves its unused slots to the fill, and a window under ``max_tokens`` terms
+    is unaffected entirely.
+
+    Recency is therefore a WEIGHT, not an absolute tier (#3760 review, Blocker
+    2). It has to be preferred at all, because the composed query is
+    ``Prior context: <older turns> … <latest>`` and a naive truncation keeps
+    whichever turn comes FIRST — i.e. throws away the actual question and keeps
+    the stalest context. But making it absolute silently defeated
+    ``recallContextTurns``: any latest turn with ``>= max_tokens`` surviving
+    terms took EVERY slot, so a conversational follow-up whose subject lives
+    only in the prior turn ("is the thing we were discussing still broken?")
+    produced a query with no subject in it at all. Step 1 is the structural
+    guarantee against that; step 3 is where recency actually decides anything,
+    and in ordinary conversational text — where nearly every candidate scores
+    0.0 on shape — that is most of the budget.
+
+    A kept compound token costs its own slot PLUS a slot for each fragment the
+    server will shred it into, so the emitted string never expands past
+    ``max_tokens`` distinct tsquery terms.
+
+    ``max_tokens <= 0`` disables shaping and returns ``query`` unchanged (the
+    operator rollback lever, ``memory.recall.query_max_tokens: 0``).
+    """
+    # Defensive coercion: `max_tokens` reaches here from settings.json /
+    # env, so a string or None is a config error, not a crash on the recall
+    # critical path. Anything uninterpretable falls back to "do not shape".
+    try:
+        max_tokens = int(max_tokens)
+    except (TypeError, ValueError):
+        return query
+    if max_tokens <= 0:
+        return query
+
+    cleaned = strip_query_scaffolding(query)
+    tokens = tokenize_for_bm25(cleaned)
+    if not tokens:
+        # Nothing tokenizable (e.g. a pure-punctuation prompt) — send the
+        # original so we never turn a real query into an empty one.
+        return query
+
+    stop = set(BM25_STOPWORDS)
+    # A bare string here (`"switchroom,agent"` mis-set in settings.json) would
+    # otherwise iterate CHARACTERS and stop-list half the alphabet.
+    if isinstance(stop_terms, str):
+        stop_terms = [t for t in re.split(r"[,\s]+", stop_terms) if t]
+    for term in stop_terms or ():
+        if isinstance(term, str) and term.strip():
+            stop.add(term.strip().lower())
+
+    latest_tokens = set(tokenize_for_bm25(strip_query_scaffolding(latest_query or "")))
+
+    first_seen = {}
+    for index, token in enumerate(tokens):
+        first_seen.setdefault(token, index)
+
+    candidates = [t for t in first_seen if len(t) > 1 and t not in stop]
+    if not candidates:
+        # Every term was a stopword. Fall back to the unfiltered token set so a
+        # short conversational prompt ("what did you say about it?") still
+        # searches for something.
+        candidates = [t for t in first_seen if len(t) > 1] or list(first_seen)
+
+    def weight(token):
+        score = _selectivity_score(token)
+        if token in latest_tokens:
+            score += _SCORE_RECENCY
+        return score
+
+    # MERIT RESERVE, then RECENCY FILL.
+    #
+    # `max_tokens // 3` slots are reserved for the highest-merit terms in the
+    # WHOLE window, scored with recency deliberately excluded, so the most
+    # discriminating terms present cannot be displaced by sheer latest-turn
+    # volume no matter which turn they came from. That is the Blocker 2
+    # guarantee, and it is structural: a conversational follow-up whose subject
+    # lives only in the prior turn ("is the thing we were discussing still
+    # broken?") keeps its subject.
+    #
+    # The remaining two thirds are filled by the recency-WEIGHTED score, which
+    # is where `_SCORE_RECENCY` earns its keep: between terms of equal merit —
+    # and after stopword removal most conversational terms are equal on merit —
+    # the one the user just typed wins, instead of the composed string's leading
+    # (i.e. STALEST) turn winning on `first_seen` alone.
+    #
+    # The reserve is a ceiling, not an allocation: any term it reserves that the
+    # recency fill would have chosen anyway costs nothing, and a window with
+    # fewer than `max_tokens` terms is unaffected entirely.
+    reserve_size = max(1, max_tokens // 3)
+    merit_ordered = sorted(candidates, key=lambda t: (-_selectivity_score(t), first_seen[t]))
+    reserved = merit_ordered[:reserve_size]
+    reserved_set = set(reserved)
+
+    # ...and a mirror-image LATEST-TURN reserve of the same size, because a
+    # merit reserve alone is one-sided. A prior turn dense in high-merit tokens
+    # (a pasted stack trace, a list of ids) outscores an ordinary question on
+    # every slot, which would cost the user the thing they actually just asked.
+    # Same shape as above: a ceiling of `max_tokens // 3`, never an allocation.
+    latest_reserve = sorted(
+        (t for t in candidates if t in latest_tokens and t not in reserved_set),
+        key=lambda t: (-weight(t), first_seen[t]),
+    )[:reserve_size]
+    reserved_set.update(latest_reserve)
+
+    filled = sorted(
+        (t for t in candidates if t not in reserved_set),
+        key=lambda t: (-weight(t), first_seen[t]),
+    )
+    candidates = reserved + latest_reserve + filled
+
+    kept = []
+    emitted = set()
+    for token in candidates:
+        # A compound token is emitted by the server ALONGSIDE its fragments;
+        # charge the budget for both so `max_tokens` is a true tsquery bound.
+        expansion = {token}
+        if _COMPOUND_TOKEN_RE.fullmatch(token):
+            expansion |= set(re.sub(r"[^\w\s]", " ", token).split())
+        new_terms = expansion - emitted
+        if len(emitted) + len(new_terms) > max_tokens:
+            continue
+        emitted |= new_terms
+        kept.append(token)
+
+    if not kept:
+        return query
+
+    kept.sort(key=lambda t: first_seen[t])
+    # Emit the ORIGINAL surface form of each survivor, not the lowercased
+    # token. BM25 lowercases and stems on the server either way, but the SAME
+    # string also feeds the embedding arm, and `Python` / `Coolify` / `PR`
+    # carry case a sentence-transformer legitimately uses.
+    surfaces = _surface_forms(cleaned)
+    return " ".join(surfaces.get(t, t) for t in kept)
+
+
+def _surface_forms(text: str) -> dict:
+    """Map each BM25 token to its first original-case spelling in ``text``."""
+    forms = {}
+    for word in re.sub(r"[^\w\s]", " ", text).split():
+        forms.setdefault(word.lower(), word)
+    for match in _COMPOUND_TOKEN_RE.finditer(text):
+        forms.setdefault(match.group(0).lower(), match.group(0))
+    return forms
+
+
 # ---------------------------------------------------------------------------
 # Turn slicing
 # ---------------------------------------------------------------------------