switchroom 0.19.41 → 0.19.43

package/vendor/hindsight-memory/scripts/tests/data/replay_volume_gate_3994.py

@@ -0,0 +1,295 @@
+#!/usr/bin/env python3
+"""Replay harness for the #3994 sub-agent volume-gate recalibration.
+
+WHAT THIS ANSWERS
+-----------------
+The sidechain retain now counts, in its volume gate, ONLY the chars the
+text-only retain path keeps (``subagent_retain.retained_text_char_count`` ->
+``lib.content._extract_text_content``). The previous gate summed text PLUS
+``tool_use`` name+input serialized size — chars the payload no longer contains.
+This harness re-measures the ``MIN_NON_TOOL_RESULT_CHARS`` floor against the
+CORRECTED metric and shows, per transcript:
+
+  * old_metric   — pre-#3994 count (text + tool_use name+input)
+  * new_metric   — post-#3994 count (text-only, == what is retained)
+  * payload_chars— the ACTUAL text-only formatted payload size
+                   (``prepare_retention_transcript(..., include_tool_calls=False)``)
+  * human_turns  — genuine human turns (``count_human_turns``)
+  * gate decision at a swept set of candidate floors, old vs new
+
+The headline the recalibration turns on: how many tool-heavy / prose-light
+forks the OLD gate wrongly PASSED (clearing on tool volume) that the NEW gate
+correctly SKIPS, while NO real worker (substantial text-only payload) is newly
+skipped at the chosen floor.
+
+DATA
+----
+Two sources, ``--dir`` preferred:
+
+  * ``--dir <path>``: replay real ``*.jsonl`` sidechain transcripts (one Claude
+    Code sidechain per file). Fleet transcripts are NOT committed to this repo
+    (they are private agent memory), so this is how an operator re-runs the
+    measurement against ground truth.
+  * default (no ``--dir``): a SYNTHETIC corpus parameterized by the empirical
+    distributions already measured and cited in ``subagent_retain.py`` /
+    ``test_subagent_retain_learnings.py`` — real klanker sidechain text-only
+    sizes (p10 5,035 / p50 10,058 / p90 26,040 chars; 0 empty, 0 < 500 over 200
+    transcripts) plus the trivial-fork and tool-heavy/prose-light shapes the
+    gate exists to reject. Deterministic (fixed seed) so the committed report is
+    reproducible.
+
+Run:  python3 scripts/tests/data/replay_volume_gate_3994.py [--dir DIR] [--json]
+
+The committed narrative report lives at
+``docs/measurements/subagent-volume-gate-3994.md`` and is regenerated from this
+harness's output.
+"""
+
+import argparse
+import json
+import os
+import random
+import sys
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+SCRIPTS_DIR = os.path.abspath(os.path.join(HERE, "..", ".."))
+if SCRIPTS_DIR not in sys.path:
+    sys.path.insert(0, SCRIPTS_DIR)
+
+import subagent_retain  # noqa: E402
+from lib.content import (  # noqa: E402
+    _extract_message_blocks,
+    prepare_retention_transcript,
+)
+
+CANDIDATE_FLOORS = [500, 1000, 1500, 2000, 3000, 5000]
+SHIPPED_FLOOR = subagent_retain.MIN_NON_TOOL_RESULT_CHARS  # 2000
+MIN_TURNS = subagent_retain.MIN_HUMAN_TURNS  # 6
+
+# A payload this size or larger is a "real worker" whose learnings we must not
+# lose. Set well under the measured klanker p10 (5,035) so the "newly skipped
+# real worker" check is strict, not self-serving.
+REAL_WORKER_PAYLOAD_FLOOR = 2000
+
+
+def _old_metric(messages):
+    """Pre-#3994 count: text + tool_use name+input serialized size."""
+    total = 0
+    for m in messages:
+        if not isinstance(m, dict):
+            continue
+        for b in _extract_message_blocks(m.get("content", ""), role=m.get("role", "")):
+            if not isinstance(b, dict) or b.get("type") == "tool_result":
+                continue
+            if b.get("type") == "text":
+                total += len(b.get("text", ""))
+            elif b.get("type") == "tool_use":
+                total += len(b.get("name", "")) + len(
+                    json.dumps(b.get("input", {}), ensure_ascii=False)
+                )
+    return total
+
+
+def _payload_chars(messages):
+    """Actual text-only formatted payload size (what is retained)."""
+    text, _ = prepare_retention_transcript(
+        messages, ["user", "assistant"], True, include_tool_calls=False
+    )
+    return len(text or "")
+
+
+def measure(messages):
+    new_metric = subagent_retain.retained_text_char_count(messages)
+    return {
+        "human_turns": subagent_retain.count_human_turns(messages),
+        "old_metric": _old_metric(messages),
+        "new_metric": new_metric,
+        "payload_chars": _payload_chars(messages),
+    }
+
+
+# ---------------------------------------------------------------------------
+# Synthetic corpus (deterministic) — models the documented distributions.
+# ---------------------------------------------------------------------------
+
+def _user(text):
+    return {"role": "user", "content": text}
+
+
+def _assistant_prose(text, *, tool_bulk=0):
+    content = [{"type": "text", "text": text}]
+    if tool_bulk:
+        content.append(
+            {"type": "tool_use", "name": "Bash",
+             "input": {"command": "grep -rn x " + ("q" * tool_bulk)}}
+        )
+    return {"role": "assistant", "content": content}
+
+
+def _tool_result_user(bulk):
+    return {"role": "user",
+            "content": [{"type": "tool_result", "tool_use_id": "t", "content": "Z" * bulk}]}
+
+
+def _real_worker(target_text_chars, rng):
+    """A genuine worker: 6-14 human turns, prose totalling ~target text chars,
+    interleaved with heavy tool traffic (the payload drops the tool traffic)."""
+    turns = rng.randint(6, 14)
+    per = max(80, target_text_chars // turns)
+    msgs = []
+    for i in range(turns):
+        msgs.append(_user(f"step {i}: continue the investigation and report findings"))
+        prose = (f"Finding {i}: confirmed the guard only probes observation rows; "
+                 "world/experience facts never traverse the semantic dedup path. ")
+        prose = (prose * ((per // len(prose)) + 1))[:per]
+        msgs.append(_assistant_prose(prose, tool_bulk=rng.randint(2000, 8000)))
+        msgs.append(_tool_result_user(rng.randint(2000, 6000)))
+    return msgs
+
+
+def _trivial_fork(rng):
+    """A 10-second fork: 2-4 turns, almost no prose. Both gates should SKIP."""
+    turns = rng.randint(2, 4)
+    msgs = []
+    for i in range(turns):
+        msgs.append(_user("go"))
+        msgs.append(_assistant_prose("ok", tool_bulk=rng.randint(0, 500)))
+    return msgs
+
+
+def _tool_heavy_prose_light(rng):
+    """The recalibration target: clears MIN_TURNS human turns and emits a LOT of
+    tool traffic, but almost NO retainable prose. OLD gate passes (tool chars),
+    NEW gate skips (text-only ~0). Retaining it stores a near-empty document."""
+    turns = rng.randint(6, 10)
+    msgs = []
+    for i in range(turns):
+        msgs.append(_user("go"))  # genuine human turn, but tiny
+        # No text block at all — pure tool_use with a large command body.
+        msgs.append({"role": "assistant",
+                     "content": [{"type": "tool_use", "name": "Bash",
+                                  "input": {"command": "true " + ("x" * rng.randint(3000, 9000))}}]})
+        msgs.append(_tool_result_user(rng.randint(3000, 8000)))
+    return msgs
+
+
+def synthetic_corpus(seed=1994):
+    rng = random.Random(seed)
+    corpus = []
+    # Real workers across the measured size range (p10..p90+).
+    for _ in range(120):
+        target = int(rng.triangular(1500, 30000, 10058))  # low, high, mode≈p50
+        corpus.append(("real_worker", _real_worker(target, rng)))
+    for _ in range(60):
+        corpus.append(("trivial_fork", _trivial_fork(rng)))
+    for _ in range(40):
+        corpus.append(("tool_heavy_prose_light", _tool_heavy_prose_light(rng)))
+    return corpus
+
+
+def _load_dir(path):
+    corpus = []
+    for name in sorted(os.listdir(path)):
+        if not name.endswith(".jsonl"):
+            continue
+        msgs = subagent_retain.read_transcript(os.path.join(path, name))
+        corpus.append((name, msgs))
+    return corpus
+
+
+def run(corpus):
+    rows = []
+    for label, msgs in corpus:
+        m = measure(msgs)
+        m["label"] = label
+        rows.append(m)
+
+    def gate(row, floor):
+        return row["human_turns"] >= MIN_TURNS and row["new_metric"] >= floor
+
+    def old_gate(row, floor):
+        return row["human_turns"] >= MIN_TURNS and row["old_metric"] >= floor
+
+    real = [r for r in rows if r["payload_chars"] >= REAL_WORKER_PAYLOAD_FLOOR]
+    empty_ish = [r for r in rows if r["payload_chars"] < 500]
+
+    floor_sweep = {}
+    for f in CANDIDATE_FLOORS:
+        newly_skipped_real = [
+            r for r in real if old_gate(r, SHIPPED_FLOOR) and not gate(r, f)
+        ]
+        # Tool-heavy/prose-light forks the OLD gate passed that NEW gate skips.
+        old_pass_new_skip_emptyish = [
+            r for r in empty_ish if old_gate(r, SHIPPED_FLOOR) and not gate(r, f)
+        ]
+        floor_sweep[f] = {
+            "passes_new": sum(1 for r in rows if gate(r, f)),
+            "real_workers_newly_skipped_vs_old_2000": len(newly_skipped_real),
+            "emptyish_forks_now_correctly_skipped": len(old_pass_new_skip_emptyish),
+        }
+
+    summary = {
+        "corpus_size": len(rows),
+        "shipped_floor": SHIPPED_FLOOR,
+        "min_turns": MIN_TURNS,
+        "counts_by_label": _counts(rows),
+        "payload_percentiles_all": _pcts([r["payload_chars"] for r in rows]),
+        "payload_percentiles_gate_passers_new_2000": _pcts(
+            [r["payload_chars"] for r in rows if gate(r, SHIPPED_FLOOR)]
+        ),
+        "old_gate_passed_but_payload_under_500": sum(
+            1 for r in rows if old_gate(r, SHIPPED_FLOOR) and r["payload_chars"] < 500
+        ),
+        "new_gate_passed_but_payload_under_500": sum(
+            1 for r in rows if gate(r, SHIPPED_FLOOR) and r["payload_chars"] < 500
+        ),
+        "floor_sweep": floor_sweep,
+    }
+    return summary, rows
+
+
+def _counts(rows):
+    out = {}
+    for r in rows:
+        out[r["label"]] = out.get(r["label"], 0) + 1
+    return out
+
+
+def _pcts(values):
+    if not values:
+        return {}
+    s = sorted(values)
+
+    def p(q):
+        return s[min(len(s) - 1, int(q * len(s)))]
+
+    return {"n": len(s), "min": s[0], "p10": p(0.10), "p50": p(0.50),
+            "p90": p(0.90), "max": s[-1]}
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--dir", help="directory of real *.jsonl sidechain transcripts")
+    ap.add_argument("--json", action="store_true", help="emit JSON only")
+    args = ap.parse_args()
+
+    if args.dir:
+        corpus = _load_dir(args.dir)
+        source = f"real transcripts from {args.dir}"
+    else:
+        corpus = synthetic_corpus()
+        source = "synthetic corpus (distribution-parameterized, seed=1994)"
+
+    summary, _ = run(corpus)
+    summary["data_source"] = source
+
+    if args.json:
+        print(json.dumps(summary, indent=2))
+        return
+
+    print(f"# Volume-gate recalibration replay (#3994)\nsource: {source}\n")
+    print(json.dumps(summary, indent=2))
+
+
+if __name__ == "__main__":
+    main()

package/vendor/hindsight-memory/scripts/tests/test_backfill_from_logs.py

@@ -58,11 +58,21 @@ class FakeDaemon:
         self.max_inflight_seen = 0
         self._inflight = 0
         self.membership_error = False
+        # #3291 partial-failure simulation: raise on the retain of the Nth
+        # DISTINCT document_id seen (distinct index, POST order = slice order),
+        # so a slice fails persistently (all bounded retries) without a fixed id.
+        self.fail_distinct_indices = set()
+        self._seen_doc_order = []
 
     def retain(self, bank_id, content, document_id="conversation", context=None,
                metadata=None, tags=None, timeout=15, async_processing=True,
                observation_scopes=None):
         self.observation_scopes_seen.append(observation_scopes)
+        if document_id not in self._seen_doc_order:
+            self._seen_doc_order.append(document_id)
+        distinct_idx = self._seen_doc_order.index(document_id)
+        if distinct_idx in self.fail_distinct_indices:
+            raise RuntimeError(f"simulated POST failure for slice #{distinct_idx}")
         self._inflight += 1
         self.max_inflight_seen = max(self.max_inflight_seen, self._inflight)
         try:
@@ -79,6 +89,12 @@ class FakeDaemon:
         return {did for did, d in self.docs.items()
                 if d.get("bank_id") == bank_id and session_id in did}
 
+    def document_exists(self, bank_id, document_id, timeout=30):
+        """Tri-state presence check (mirrors HindsightClient.document_exists):
+        True when the slice doc is durable, False on a confirmed miss."""
+        d = self.docs.get(document_id)
+        return bool(d is not None and d.get("bank_id") == bank_id)
+
 
 def _write_registry(path, rows, with_session_id=False):
     os.makedirs(os.path.dirname(path), exist_ok=True)
@@ -166,6 +182,7 @@ class FromLogsTestBase(unittest.TestCase):
         self._patches = [
             mock.patch.object(HindsightClient, "retain", self._fake_retain),
             mock.patch.object(HindsightClient, "list_session_document_ids", self._fake_membership),
+            mock.patch.object(HindsightClient, "document_exists", self._fake_doc_exists),
             mock.patch("backfill_transcripts.get_api_url", return_value="http://fake"),
             mock.patch("backfill_transcripts._SLEEP", self._fake_sleep),
         ]
@@ -178,6 +195,9 @@ class FromLogsTestBase(unittest.TestCase):
     def _fake_membership(self, *a, **kw):
         return self.daemon.list_session_document_ids(*a, **kw)
 
+    def _fake_doc_exists(self, *a, **kw):
+        return self.daemon.document_exists(*a, **kw)
+
     def _fake_sleep(self, secs):
         self.sleeps.append(secs)
 
@@ -448,6 +468,95 @@ class TestFromLogs(FromLogsTestBase):
         spaced = [s for s in self.sleeps if abs(s - 1.5) < 1e-9]
         self.assertEqual(len(spaced), 3)
 
+    # --- #3291: partial POST failure then re-run must GAP-COMPLETE, not skip --
+    #     Run 1: slices 0..k-1 commit (async_processing=False ⇒ durable), slice k
+    #     fails ⇒ session recorded ``failed``. Run 2 (the fix): the committed
+    #     prefix must NOT classify the session ``present`` and skip it forever —
+    #     it must recompute the deterministic slice ids and POST ONLY the missing
+    #     slice(s). Pre-fix behaviour: run 2 sees the prefix via session
+    #     membership → ``has_documents`` → skip → the tail is lost silently. -----
+    def test_partial_post_failure_then_rerun_gap_completes(self):
+        self._write_yaml({"clerk": None})
+        self._write_settings("clerk", "clerk")
+        base = 1_700_000_000_000
+        # 3 human turns, slice_turns=1 ⇒ 3 non-overlapping slices posted in order.
+        path, (first, last) = self._transcript("clerk", "sess-partial", 3, base)
+        self._registry("clerk", [{
+            "turn_key": "123:_:t", "chat_id": "123", "started_at": str(first + 100),
+            "ended_at": str(last), "ended_via": "sigterm",
+        }])
+
+        # RUN 1 — the LAST slice (distinct index 2) fails every (bounded) attempt.
+        self.daemon.fail_distinct_indices = {2}
+        report1 = self._run("clerk")
+        ar1 = report1["agents"]["clerk"]
+
+        # Total-loss restore attempted all 3 slices; 2 committed, 1 failed.
+        self.assertEqual(ar1["sessions_total_loss"], 1)
+        self.assertEqual(ar1["sessions_restored"], 0)   # NOT fully restored
+        self.assertEqual(ar1["posts_ok"], 2)
+        self.assertEqual(ar1["posts_failed"], 1)
+        entry1 = next(s for s in ar1["sessions"] if s["class"] == "total_loss")
+        all_ids = entry1["document_ids"]
+        self.assertEqual(len(all_ids), 3)
+        committed_after_1 = set(self.daemon.docs)
+        self.assertEqual(len(committed_after_1), 2)     # durable committed prefix
+        missing = [d for d in all_ids if d not in committed_after_1]
+        self.assertEqual(len(missing), 1)
+        # Progress marked failed (NOT done) — the re-run must revisit it.
+        with open(os.environ["HINDSIGHT_BACKFILL_PROGRESS"]) as f:
+            prog = json.load(f)
+        self.assertEqual(prog["clerk/sess-partial"]["status"], "failed")
+
+        # RUN 2 — no injected failure this time. The fix must gap-complete.
+        self.daemon.fail_distinct_indices = set()
+        posts_before = len(self.daemon.posts)
+        report2 = self._run("clerk")
+        ar2 = report2["agents"]["clerk"]
+
+        # It must NOT be skipped as an already-present session.
+        self.assertEqual(ar2["sessions_has_documents"], 0)
+        # Gap completion: exactly the missing slice posted, the 2 present skipped.
+        self.assertEqual(ar2["sessions_gap_completed"], 1)
+        self.assertEqual(ar2["slices_gap_filled"], 1)
+        self.assertEqual(ar2["slices_gap_present"], 2)
+        run2_posts = [did for did, _ in self.daemon.posts[posts_before:]]
+        self.assertEqual(run2_posts, missing)           # ONLY the missing slice
+        # All three slices are now durable — the session is fully recovered.
+        self.assertEqual(set(self.daemon.docs), set(all_ids))
+        with open(os.environ["HINDSIGHT_BACKFILL_PROGRESS"]) as f:
+            prog2 = json.load(f)
+        self.assertEqual(prog2["clerk/sess-partial"]["status"], "done")
+        # Watermark advanced to the true tail slice on completion.
+        self.assertIsNotNone(watermark.load("sess-partial"))
+
+    # --- #3291 self-heal: a persistently-failing slice must surface as an
+    #     operator-visible INCOMPLETE metric + convergence WARN, never a silent
+    #     trap. -----------------------------------------------------------------
+    def test_incomplete_session_surfaces_convergence_metric(self):
+        self._write_yaml({"clerk": None})
+        self._write_settings("clerk", "clerk")
+        base = 1_700_000_000_000
+        path, (first, last) = self._transcript("clerk", "sess-stuck", 2, base)
+        self._registry("clerk", [{
+            "turn_key": "123:_:t", "chat_id": "123", "started_at": str(first + 100),
+            "ended_at": str(last), "ended_via": "sigterm",
+        }])
+        # The first slice fails every attempt → session left INCOMPLETE.
+        self.daemon.fail_distinct_indices = {0}
+
+        report = self._run("clerk")
+        ar = report["agents"]["clerk"]
+        # Metric: one session did not fully recover.
+        self.assertEqual(ar["sessions_incomplete"], 1)
+        self.assertEqual(ar["sessions_restored"], 0)
+        self.assertEqual(report["rollup"]["sessions_incomplete"], 1)
+        # Operator-visible: the formatted report carries the count + a WARN that
+        # tells them to re-run (so a non-converging session is never silent).
+        text = bf.format_report_from_logs(report)
+        self.assertIn("incomplete=1", text)
+        self.assertIn("CONVERGENCE", text)
+
     # --- direct session_id join (migrated registry) -------------------------
     def test_direct_sessionid_join(self):
         self._write_yaml({"clerk": None})

package/vendor/hindsight-memory/scripts/tests/test_overlap_tokens.py

@@ -0,0 +1,135 @@
+"""Characterisation of ``recall._overlap_tokens`` — the transcript-fallback tokenizer.
+
+WHY THIS FILE EXISTS
+--------------------
+``_overlap_tokens`` is the keyword tokenizer behind the #3369 transcript-grep
+recall fallback: a recent transcript turn is injected only if its token set
+intersects the query's (``recall._build_transcript_fallback``). When the
+lexical-overlap recall GATE was removed (#3761, commit e3b9d62f) the gate's
+characterisation tests went with it — but ``_overlap_tokens`` itself stayed
+load-bearing for the fallback. That left a real, shipped tokenizer with no
+direct unit coverage: a change to its rules (letters-only vs alphanumeric, the
+length floor, stopword handling) could silently move which transcript turns
+recall can surface, and nothing would fail.
+
+These tests pin the tokenizer's contract directly (#3777), including the #3578
+decision that DIGITS CARRY SIGNAL (issue numbers, ports, versions). They assert
+OUTCOMES of the tokenizer, not code paths, and each would fail on a real
+regression of the documented behaviour.
+
+Stdlib-only; runs under ``python3 -m unittest discover tests/``.
+"""
+
+import os
+import sys
+import unittest
+
+SCRIPTS_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+if SCRIPTS_DIR not in sys.path:
+    sys.path.insert(0, SCRIPTS_DIR)
+
+import recall  # noqa: E402
+
+tok = recall._overlap_tokens
+
+
+class OverlapTokensCharacterisation(unittest.TestCase):
+    """Pure-function contract for ``recall._overlap_tokens`` (#3777)."""
+
+    # --- empty / non-string inputs -------------------------------------------
+
+    def test_empty_string_is_empty_set(self):
+        self.assertEqual(tok(""), set())
+
+    def test_none_is_empty_set(self):
+        self.assertEqual(tok(None), set())
+
+    def test_non_string_scalar_is_empty_set(self):
+        self.assertEqual(tok(123), set())
+
+    def test_non_string_container_is_empty_set(self):
+        self.assertEqual(tok(["deploy", "server"]), set())
+        self.assertEqual(tok({"deploy": 1}), set())
+
+    def test_whitespace_only_is_empty_set(self):
+        self.assertEqual(tok("   \t\n "), set())
+
+    # --- basic tokenisation ---------------------------------------------------
+
+    def test_simple_words_split_on_whitespace(self):
+        self.assertEqual(tok("deploy staging server"), {"deploy", "staging", "server"})
+
+    def test_lowercased(self):
+        self.assertEqual(tok("Deploy STAGING Server"), {"deploy", "staging", "server"})
+
+    def test_case_folds_to_a_single_token(self):
+        self.assertEqual(tok("DEPLOY deploy Deploy"), {"deploy"})
+
+    def test_duplicates_collapse_to_a_set(self):
+        self.assertEqual(tok("server server server"), {"server"})
+
+    def test_trailing_token_without_separator_is_flushed(self):
+        # The final run has no trailing delimiter; it must still be emitted.
+        self.assertEqual(tok("deploy server"), {"deploy", "server"})
+
+    # --- punctuation as separators -------------------------------------------
+
+    def test_punctuation_is_a_separator(self):
+        self.assertEqual(tok("deploy, the staging server!"), {"deploy", "staging", "server"})
+
+    def test_hyphen_splits_tokens(self):
+        self.assertEqual(tok("multi-word thing"), {"multi", "word", "thing"})
+
+    def test_underscore_splits_tokens(self):
+        # '_' is not alphanumeric, so it is a separator (documented behaviour).
+        self.assertEqual(tok("foo_bar baz"), {"foo", "bar", "baz"})
+
+    def test_dotted_version_splits_on_dots(self):
+        self.assertEqual(tok("v0.19.24"), {"v0", "19", "24"})
+
+    # --- length floor ---------------------------------------------------------
+
+    def test_single_alpha_char_is_dropped(self):
+        self.assertEqual(tok("x y z deploy"), {"deploy"})
+
+    def test_single_digit_char_is_dropped_as_noise(self):
+        # A lone digit is noise, not an identifier — dropped by the > 1 guard,
+        # exactly like a lone letter.
+        self.assertEqual(tok("python 9 angular"), {"python", "angular"})
+
+    def test_two_char_token_survives_the_floor(self):
+        self.assertEqual(tok("pr go"), {"pr", "go"})
+
+    # --- stopwords ------------------------------------------------------------
+
+    def test_stopwords_removed(self):
+        self.assertEqual(tok("the is a to of"), set())
+
+    def test_content_survives_with_stopwords_present(self):
+        self.assertEqual(tok("what is the deploy status"), {"deploy", "status"})
+
+    # --- #3578: digits carry signal ------------------------------------------
+
+    def test_pure_digit_issue_number_is_a_token(self):
+        self.assertEqual(tok("PR 3993"), {"pr", "3993"})
+
+    def test_hash_prefixed_issue_number_keeps_the_digits(self):
+        self.assertEqual(tok("#3541 shipped"), {"3541", "shipped"})
+
+    def test_colon_prefixed_port_keeps_the_digits(self):
+        self.assertEqual(tok(":9077 port"), {"9077", "port"})
+
+    def test_mixed_alphanumeric_run_stays_intact(self):
+        self.assertEqual(tok("abc123def"), {"abc123def"})
+
+    def test_identifier_overlap_between_query_and_transcript(self):
+        # The #3578 motivating case: a transcript turn whose only tie to the
+        # query is an issue number must now share a token (it did not before).
+        query = tok("did PR 3993 land and what did v0.19.24 change")
+        turn = tok("3993 shipped in 0.19.24 last night")
+        self.assertIn("3993", query & turn)
+        self.assertTrue(query & turn)
+
+
+if __name__ == "__main__":
+    unittest.main()

package/vendor/hindsight-memory/scripts/tests/test_recall_no_lexical_gate.py

@@ -115,21 +115,15 @@ def _run_main_with(client, prompt="what did we decide", config_extra=None):
     return json.loads(raw)["hookSpecificOutput"]["additionalContext"], raw
 
 
-class GateHelpersAreGoneTests(unittest.TestCase):
-    """The gate's machinery must not exist to be re-wired by accident."""
-
-    def test_gate_helpers_are_removed(self):
-        self.assertFalse(hasattr(recall, "containment_overlap"))
-        self.assertFalse(hasattr(recall, "_filter_by_overlap"))
-
-    def test_tokenizer_survives_for_the_transcript_fallback(self):
-        """`_overlap_tokens` is still load-bearing for
-        `_build_transcript_fallback`'s keyword match — deleting it with the
-        gate would silently break the #3369 fallback."""
-        self.assertEqual(
-            recall._overlap_tokens("Deploy the staging server!"),
-            {"deploy", "staging", "server"},
-        )
+#
+# NOTE (#3999): the former ``GateHelpersAreGoneTests`` hasattr guard
+# (``assertFalse(hasattr(recall, "containment_overlap"))``) was dropped as
+# redundant — a re-added-but-unwired helper does nothing, and a re-WIRED gate is
+# already caught by the outcome assertions in ``NoLexicalDroppingIntegrationTests``
+# below (which fail if any candidate is dropped for lexical reasons). Direct
+# characterisation of the surviving ``_overlap_tokens`` tokenizer now lives in
+# ``test_overlap_tokens.py`` (#3777), not in a hasattr check here.
+#
 
 
 class NoLexicalDroppingIntegrationTests(unittest.TestCase):
@@ -151,17 +145,20 @@ class NoLexicalDroppingIntegrationTests(unittest.TestCase):
         self.assertIn("vegan dinner recipes", ctx)
 
     def test_identifier_only_match_survives(self):
-        """The measured root cause: `_overlap_tokens` drops digits and
-        1-char tokens, so a memory whose ONLY relationship to the prompt is
-        an identifier scored 0.0 and was always discarded. Assert directly
-        that the tokenizer still cannot see the identifier, AND that the
-        memory is injected anyway."""
+        """A memory whose ONLY tie to the prompt is an identifier survives.
+
+        This was the measured root cause of the removed gate: `_overlap_tokens`
+        dropped digits, so an identifier-only memory scored 0.0 and was always
+        discarded. Two things now hold: (a) #3578 gave the tokenizer digit
+        vision, so query and memory now DO share the identifier token; and (b)
+        with the gate gone, the memory is injected regardless of overlap."""
         query = PREAMBLE + "did PR #3541 land, and what did v0.19.24 change"
         mem = "#3541 shipped in 0.19.24"
-        self.assertEqual(
+        # Post-#3578 the identifier is visible on both sides.
+        self.assertIn(
+            "3541",
             recall._overlap_tokens(query) & recall._overlap_tokens(mem),
-            set(),
-            "fixture drifted: it must share no ALPHABETIC token with the query",
+            "the shared identifier must now be a token on both sides (#3578)",
         )
         ctx, _ = _run_main_with(_FakeClient([_memory(mem, score=0.95)]), prompt=query)
         self.assertIsNotNone(ctx)