switchroom 0.19.23 → 0.19.25

package/vendor/hindsight-memory/tests/test_content.py

@@ -7,13 +7,16 @@ import pytest
 from lib.content import (
     _extract_text_content,
     _is_channel_message_tool,
+    _selectivity_score,
     compose_recall_query,
     format_current_time,
     format_memories,
     prepare_retention_transcript,
+    shape_recall_query,
     slice_last_turns_by_user_boundary,
     strip_channel_envelope,
     strip_memory_tags,
+    tokenize_for_bm25,
     truncate_recall_query,
 )
 
@@ -648,3 +651,218 @@ class TestFormatCurrentTime:
         monkeypatch.delenv("TZ", raising=False)
         out = format_current_time()
         assert re.match(r"\d{4}-\d{2}-\d{2} \d{2}:\d{2} (?:AM|PM)", out), out
+
+
+# ---------------------------------------------------------------------------
+# Switchroom #3757 — BM25 query shaping
+#
+# The bug these guard: the recall hook composed an 800-char, ~110-token query
+# and Hindsight OR-joined every token into one tsquery. On the live `overlord`
+# bank that matched 119,510 of 135,565 rows and took 14.0s for the 3-arm BM25
+# UNION — past the per-bank client timeout, so 96.8% of that agent's own-bank
+# recalls returned NOTHING. Two of those tokens were scaffolding this hook
+# added itself: `user` (67,363 rows = 50% of the bank) and `assistant`
+# (29,942 = 22%).
+# ---------------------------------------------------------------------------
+
+
+def _production_shaped_query():
+    """A composed query with the shape recall.py actually produces."""
+    msgs = _msgs(
+        ("user", "the deploy went out this morning but the rollout never reached the fleet"),
+        (
+            "assistant",
+            "I checked the manifest for v0.19.17 and the digest does not match what "
+            "the agent container is running, so the pull raced the tag",
+        ),
+    )
+    return compose_recall_query(
+        "why did recall for the v0.19.17 rollout return v0.18.15 instead",
+        msgs,
+        recall_context_turns=2,
+    )
+
+
+class TestComposedQueryHasNoRoleLabels:
+    def test_compose_does_not_prefix_role_labels(self):
+        # The regression this guards: `context_lines.append(f"{role}: {content}")`.
+        msgs = _msgs(("user", "prior question"), ("assistant", "prior answer"))
+        result = compose_recall_query("current question", msgs, recall_context_turns=2)
+        assert "prior question" in result
+        assert "prior answer" in result
+        assert "user:" not in result
+        assert "assistant:" not in result
+
+    def test_role_labels_are_not_bm25_tokens(self):
+        # The property that actually matters — not "the string is absent" but
+        # "the term never reaches the tsquery".
+        query = _production_shaped_query()
+        shaped = shape_recall_query(query, "why did recall for the v0.19.17 rollout "
+                                           "return v0.18.15 instead")
+        terms = set(tokenize_for_bm25(shaped))
+        assert "user" not in terms
+        assert "assistant" not in terms
+        assert "prior" not in terms
+        assert "context" not in terms
+
+    def test_labels_embedded_in_transcript_text_are_still_stripped(self):
+        # Defence in depth: an old composed string, or a turn that literally
+        # contains a `user:` line, must not smuggle the label back in.
+        legacy = "Prior context:\n\nuser: the deploy failed\nassistant: I checked it\n\nwhy"
+        terms = set(tokenize_for_bm25(shape_recall_query(legacy, "why")))
+        assert "user" not in terms
+        assert "assistant" not in terms
+        assert "deploy" in terms
+
+
+class TestShapeRecallQuery:
+    def test_caps_distinct_bm25_terms(self):
+        query = " ".join(f"distinctword{i}" for i in range(200))
+        shaped = shape_recall_query(query, "", max_tokens=24)
+        assert len(set(tokenize_for_bm25(shaped))) <= 24
+
+    def test_cap_counts_compound_token_expansion(self):
+        # A compound is emitted by the server ALONGSIDE its fragments
+        # (`v0.19.17` → v0.19.17, v0, 19, 17), so a naive "count the words we
+        # sent" cap would silently ship 4x the terms it promised.
+        query = " ".join(f"v0.19.{i}" for i in range(40))
+        shaped = shape_recall_query(query, "", max_tokens=24)
+        assert len(set(tokenize_for_bm25(shaped))) <= 24
+
+    def test_cap_is_configurable(self):
+        query = " ".join(f"distinctword{i}" for i in range(200))
+        assert len(set(tokenize_for_bm25(shape_recall_query(query, "", max_tokens=8)))) <= 8
+        assert len(set(tokenize_for_bm25(shape_recall_query(query, "", max_tokens=40)))) <= 40
+
+    def test_zero_disables_shaping(self):
+        query = "Prior context:\n\nsomething\n\nlatest"
+        assert shape_recall_query(query, "latest", max_tokens=0) == query
+
+    def test_prefers_latest_turn_over_prior_context(self):
+        # A chronological truncation would keep the OLDEST context and throw
+        # away the question the user actually asked, so the latest turn takes
+        # the MAJORITY of the budget. But recency is a weight plus a bounded
+        # quota, not an absolute tier (#3760 review, Blocker 2): the subject of
+        # a conversation routinely sits in the turn BEFORE the one that refers
+        # to it as "it"/"that", and an absolute tier starves it every time.
+        prior = " ".join(f"stalecontextword{i}" for i in range(60))
+        latest = "why does the reaper skip orphaned worktrees"
+        query = f"Prior context:\n\n{prior}\n\n{latest}"
+        # Every prior token here is strictly MORE selective than anything in
+        # the question (they carry digits; the question is plain English), so
+        # pure merit ordering would hand prior context all four slots and the
+        # user would search for none of what they asked.
+        assert _selectivity_score("stalecontextword0") > _selectivity_score("worktrees")
+        # The latest-turn reserve is what stops that. At a punishing cap it is
+        # only `max_tokens // 3` slots — the question is represented, not
+        # preserved whole, which is the honest tradeoff when prior context is
+        # genuinely more discriminating.
+        tight = tokenize_for_bm25(shape_recall_query(query, latest, max_tokens=4))
+        from_question = set(tight) & {"reaper", "skip", "orphaned", "worktrees"}
+        assert len(from_question) >= max(1, 4 // 3)
+        assert len(set(tight)) <= 4
+        # With slack, the latest turn is still fully present and the leftover
+        # budget goes to context (which is the point of composing at all).
+        loose = set(tokenize_for_bm25(shape_recall_query(query, latest, max_tokens=12)))
+        assert {"reaper", "skip", "orphaned", "worktrees"} <= loose
+        assert any(t.startswith("stalecontextword") for t in loose)
+        assert len(loose) <= 12
+
+    def test_drops_english_stopwords(self):
+        latest = "what did we decide about the worktree reaper"
+        terms = set(tokenize_for_bm25(shape_recall_query(latest, latest, max_tokens=24)))
+        assert "worktree" in terms
+        assert "reaper" in terms
+        assert "decide" in terms
+        for stop in ("what", "did", "we", "about", "the"):
+            assert stop not in terms
+
+    def test_operator_stop_terms_are_dropped(self):
+        # Bank-specific high-df words the generic stoplist cannot know about.
+        latest = "the switchroom agent rollout stalled on the reaper"
+        terms = set(
+            tokenize_for_bm25(
+                shape_recall_query(latest, latest, max_tokens=24,
+                                   stop_terms=["switchroom", "agent"])
+            )
+        )
+        assert "reaper" in terms
+        assert "rollout" in terms
+        assert "switchroom" not in terms
+        assert "agent" not in terms
+
+    def test_short_query_survives_intact(self):
+        # No regression for the small-bank / short-prompt case: every content
+        # word is kept, so recall quality is unchanged.
+        latest = "worktree gc reaper timings"
+        terms = set(tokenize_for_bm25(shape_recall_query(latest, latest, max_tokens=24)))
+        assert terms == {"worktree", "gc", "reaper", "timings"}
+
+    def test_all_stopword_query_is_not_emptied(self):
+        # A conversational prompt must never be shaped down to nothing — an
+        # empty query would return zero memories, the exact failure we are
+        # fixing.
+        latest = "what about that"
+        shaped = shape_recall_query(latest, latest, max_tokens=24)
+        assert tokenize_for_bm25(shaped)
+
+    def test_untokenizable_query_returns_original(self):
+        assert shape_recall_query("!!! ???", "!!! ???", max_tokens=24) == "!!! ???"
+
+    def test_preserves_original_word_order(self):
+        latest = "reaper skipped orphaned worktrees before rollout"
+        shaped = shape_recall_query(latest, latest, max_tokens=24)
+        # "before" is a stopword; the survivors keep the source order.
+        assert shaped.split() == ["reaper", "skipped", "orphaned", "worktrees", "rollout"]
+
+    def test_unparseable_cap_disables_shaping_instead_of_raising(self):
+        # max_tokens arrives from settings.json / env. A config error must
+        # degrade to "send it unshaped", never raise on the recall hot path.
+        latest = "the reaper skipped orphaned worktrees before the rollout"
+        for bad in (None, "x", "", [], {}):
+            assert shape_recall_query(latest, latest, max_tokens=bad) == latest
+
+    def test_numeric_string_cap_is_honoured(self):
+        latest = "the reaper skipped orphaned worktrees before the rollout"
+        shaped = shape_recall_query(latest, latest, max_tokens="3")
+        assert len(set(tokenize_for_bm25(shaped))) <= 3
+
+    def test_stop_terms_given_as_a_bare_string_are_split_not_iterated(self):
+        # "reaper,worktrees" iterated as characters would stop-list half the
+        # alphabet and gut the query.
+        latest = "the reaper skipped orphaned worktrees before the rollout"
+        shaped = shape_recall_query(latest, latest, 24, stop_terms="reaper, worktrees")
+        terms = set(tokenize_for_bm25(shaped))
+        assert "reaper" not in terms
+        assert "worktrees" not in terms
+        assert {"skipped", "orphaned", "rollout"} <= terms
+
+    def test_preserves_original_case(self):
+        # The shaped string feeds BOTH arms. BM25 lowercases server-side, but
+        # the embedding arm does not, so shaping must not flatten `Python` to
+        # `python` or `PR` to `pr`.
+        latest = "should the Python worker open a PR against Coolify"
+        shaped = shape_recall_query(latest, latest, max_tokens=24)
+        assert shaped.split() == ["Python", "worker", "open", "PR", "Coolify"]
+
+    def test_production_shaped_query_fits_the_budget(self):
+        query = _production_shaped_query()
+        latest = "why did recall for the v0.19.17 rollout return v0.18.15 instead"
+        shaped = shape_recall_query(query, latest, max_tokens=24)
+        terms = set(tokenize_for_bm25(shaped))
+        assert len(terms) <= 24
+        # The discriminating identifiers from the latest turn survive.
+        assert "v0.19.17" in terms
+        assert "rollout" in terms
+
+
+class TestTokenizeForBm25:
+    def test_matches_server_tokenizer_on_compounds(self):
+        # Mirrors hindsight_api/engine/search/retrieval.py::tokenize_query —
+        # fragments PLUS the intact compound.
+        tokens = tokenize_for_bm25("bumped to v0.19.17")
+        assert "v0.19.17" in tokens
+        assert {"v0", "19", "17"} <= set(tokens)
+
+    def test_empty_for_punctuation_only(self):
+        assert tokenize_for_bm25("!!! ???") == []