switchroom 0.19.16 → 0.19.18

package/vendor/hindsight-memory/scripts/tests/test_recall_integration.py

@@ -433,6 +433,43 @@ class RecallTelemetryLogTests(unittest.TestCase):
         self.assertEqual(entries[0]["demoted_count"], 1)
         self.assertEqual(entries[0]["memory_ids"], ["k1"])
 
+    def test_logs_injected_score_aggregates_post_slice(self):
+        """#3541 recall-QUALITY telemetry: the row must carry min/median/max
+        of `scores.final` for the memories actually INJECTED — i.e. after the
+        head-slice, not the pre-gate candidate pool. Without this, a rollout
+        where the gate stops dropping and the cap fills every turn is
+        indistinguishable from one feeding 8 mediocre memories per turn.
+        """
+        def _scored(text, mem_id, final):
+            m = _memory(text, mem_id=mem_id)
+            m["scores"] = {"final": final}
+            return m
+
+        memories = [
+            _scored("low", "id-low", 0.10),
+            _scored("high", "id-high", 0.90),
+            _scored("mid", "id-mid", 0.80),
+        ]
+        client = _FakeClient(directives=[], memories=memories)
+        _run_main_with(client, config_extra={"recallMaxMemories": 2})
+        e = self._read_log()[0]
+        # Sorted desc then sliced → the 0.10 memory is NOT injected, and must
+        # not appear in the aggregates.
+        self.assertEqual(e["memory_ids"], ["id-high", "id-mid"])
+        self.assertAlmostEqual(e["injected_score_max"], 0.90, places=6)
+        self.assertAlmostEqual(e["injected_score_min"], 0.80, places=6)
+        self.assertAlmostEqual(e["injected_score_median"], 0.85, places=6)
+
+    def test_injected_score_aggregates_are_null_without_scores(self):
+        """Score-less results (or an empty set) must log nulls, never a
+        sentinel like -inf that would corrupt a dashboard average."""
+        client = _FakeClient(directives=[], memories=[_memory("x", mem_id="x1")])
+        _run_main_with(client)
+        e = self._read_log()[0]
+        self.assertIsNone(e["injected_score_min"])
+        self.assertIsNone(e["injected_score_median"])
+        self.assertIsNone(e["injected_score_max"])
+
     def test_no_log_when_plugin_data_unset(self):
         # If CLAUDE_PLUGIN_DATA isn't set, the writer no-ops silently —
         # we don't want a stray log file in the working directory.
@@ -509,59 +546,106 @@ class AckShortCircuitTests(unittest.TestCase):
         self.assertIsNotNone(ctx)
 
 
-class JaccardOverlapUnitTests(unittest.TestCase):
-    """Switchroom #475: pure-function tests for the relevance helpers."""
+class ContainmentOverlapUnitTests(unittest.TestCase):
+    """Switchroom #475 / #3541: pure-function tests for the relevance helpers."""
 
     def test_identical_text_is_full_overlap(self):
         # Modulo stop-word stripping (`is`, `the`, `a`, `to` removed).
         self.assertEqual(
-            recall.jaccard_overlap("deploy the staging server", "deploy the staging server"),
+            recall.containment_overlap("deploy the staging server", "deploy the staging server"),
             1.0,
         )
 
     def test_disjoint_text_is_zero(self):
         self.assertEqual(
-            recall.jaccard_overlap("deploy staging server", "vegan dinner recipes"),
+            recall.containment_overlap("deploy staging server", "vegan dinner recipes"),
             0.0,
         )
 
     def test_partial_overlap_is_between(self):
-        score = recall.jaccard_overlap(
+        score = recall.containment_overlap(
             "deploy staging server",
             "deploy production server",
         )
         # {deploy, staging, server} vs {deploy, production, server}
-        # → intersection 2, union 4 → 0.5
-        self.assertAlmostEqual(score, 0.5, places=2)
+        # → intersection 2, |M| = 3 → 0.666…
+        self.assertAlmostEqual(score, 2 / 3, places=2)
 
     def test_stopwords_dont_inflate_overlap(self):
         # "the" / "is" / "a" present in both shouldn't count.
-        score = recall.jaccard_overlap("the cat is a pet", "the dog is a pet")
+        score = recall.containment_overlap("the cat is a pet", "the dog is a pet")
         # Real tokens after stopword strip: {cat, pet} vs {dog, pet}
-        # → intersection 1, union 3 → 0.333…
-        self.assertAlmostEqual(score, 1 / 3, places=2)
+        # → intersection 1, |M| = 2 → 0.5
+        self.assertAlmostEqual(score, 0.5, places=2)
 
     def test_empty_text_yields_zero(self):
-        self.assertEqual(recall.jaccard_overlap("", "anything at all"), 0.0)
-        self.assertEqual(recall.jaccard_overlap("query", ""), 0.0)
+        self.assertEqual(recall.containment_overlap("", "anything at all"), 0.0)
+        self.assertEqual(recall.containment_overlap("query", ""), 0.0)
 
     def test_non_string_inputs_yield_zero(self):
-        self.assertEqual(recall.jaccard_overlap(None, "x"), 0.0)
-        self.assertEqual(recall.jaccard_overlap("x", None), 0.0)
+        self.assertEqual(recall.containment_overlap(None, "x"), 0.0)
+        self.assertEqual(recall.containment_overlap("x", None), 0.0)
 
     def test_case_insensitive(self):
         self.assertEqual(
-            recall.jaccard_overlap("DEPLOY Server", "deploy server"),
+            recall.containment_overlap("DEPLOY Server", "deploy server"),
             1.0,
         )
 
     def test_punctuation_stripped(self):
         self.assertEqual(
-            recall.jaccard_overlap("deploy, server!", "deploy server"),
+            recall.containment_overlap("deploy, server!", "deploy server"),
             1.0,
         )
 
 
+class InjectedScoreStatsUnitTests(unittest.TestCase):
+    """#3541 — aggregates for the injected set. Must be total and non-fatal:
+    telemetry can never take recall down."""
+
+    @staticmethod
+    def _m(final):
+        return {"text": "t", "scores": {"final": final}}
+
+    def test_odd_count_uses_middle_value(self):
+        s = recall._injected_score_stats([self._m(0.1), self._m(0.9), self._m(0.5)])
+        self.assertEqual(s["injected_score_min"], 0.1)
+        self.assertEqual(s["injected_score_median"], 0.5)
+        self.assertEqual(s["injected_score_max"], 0.9)
+
+    def test_even_count_averages_the_two_middles(self):
+        s = recall._injected_score_stats(
+            [self._m(0.2), self._m(0.4), self._m(0.6), self._m(0.8)]
+        )
+        self.assertEqual(s["injected_score_median"], 0.5)
+
+    def test_empty_set_is_all_none(self):
+        self.assertEqual(
+            recall._injected_score_stats([]),
+            {
+                "injected_score_min": None,
+                "injected_score_median": None,
+                "injected_score_max": None,
+            },
+        )
+
+    def test_scoreless_entries_are_excluded_not_treated_as_negative_infinity(self):
+        s = recall._injected_score_stats([self._m(0.6), {"text": "no scores"}])
+        self.assertEqual(s["injected_score_min"], 0.6)
+        self.assertEqual(s["injected_score_max"], 0.6)
+
+    def test_malformed_input_returns_nulls_instead_of_raising(self):
+        # A telemetry error must never propagate into the recall path.
+        self.assertEqual(
+            recall._injected_score_stats(object()),
+            {
+                "injected_score_min": None,
+                "injected_score_median": None,
+                "injected_score_max": None,
+            },
+        )
+
+
 class OverlapFilterUnitTests(unittest.TestCase):
     """Switchroom #475: _filter_by_overlap behaviour."""
 
@@ -584,17 +668,277 @@ class OverlapFilterUnitTests(unittest.TestCase):
     def test_threshold_keeps_partial_match_at_or_above(self):
         results = [_memory("deploy production server")]
         kept, dropped = recall._filter_by_overlap(results, "deploy staging server", 0.5)
-        # 2/4 = 0.5 ≥ 0.5 → kept
+        # 2/|M| = 2/3 = 0.666… ≥ 0.5 → kept
         self.assertEqual(len(kept), 1)
         self.assertEqual(dropped, 0)
 
     def test_threshold_drops_partial_match_below(self):
         results = [_memory("deploy production server")]
-        kept, dropped = recall._filter_by_overlap(results, "deploy staging server", 0.51)
+        # 2/|M| = 2/3 = 0.666… < 0.7 → dropped
+        kept, dropped = recall._filter_by_overlap(results, "deploy staging server", 0.7)
         self.assertEqual(len(kept), 0)
         self.assertEqual(dropped, 1)
 
 
+class OverlapGateQueryLengthInvarianceTests(unittest.TestCase):
+    """Switchroom #3541 regression: the gate must not be decided by prompt length.
+
+    Production telemetry (1548 recall_log.jsonl rows, ts >= 2026-07-18) showed
+    the survival rate through this gate collapsing monotonically as the prompt
+    grew — 7.3% at <200 query chars down to 0.9% at 600-750 chars — with a
+    93% zero-result rate at the long end. That is the Jaccard union term, not a
+    relevance judgement. These tests pin the containment metric that fixes it.
+    """
+
+    # A realistic recall query: the memory-bearing sentence plus the long
+    # unrelated prior-context preamble the UserPromptSubmit hook prepends.
+    CORE_QUERY = "where did we land on the postgres connection pool sizing"
+    PREAMBLE = (
+        "Prior context: the operator asked about telegram card rendering, then "
+        "about vault broker grants, then about the nightly digest schedule and "
+        "the reaction dispatch wiring, and separately about docker image "
+        "promotion, canary holdback, worktree hygiene, changelog discipline, "
+        "eval harness thresholds, skill authoring quotas and mental model "
+        "refresh cadence across the whole fleet of agents on this host. "
+    )
+    MEMORY = "Postgres connection pool sizing was settled at min 5 max 100."
+
+    @staticmethod
+    def _filler(n_words):
+        """Generate `n_words` DISTINCT unrelated content words.
+
+        Distinctness matters: the metrics operate on token *sets*, so
+        repeating the same paragraph would not grow ``|Q|`` and a
+        length-sensitivity test built on it would silently pass even under
+        the buggy Jaccard metric.
+
+        The words must also be purely alphabetic — ``_overlap_tokens``
+        splits on every non-alpha character, so a digit-suffixed word like
+        ``filler7word`` collapses to the two tokens ``filler``/``word`` no
+        matter how many are emitted.
+        """
+        letters = "abcdefghijklmnopqrstuvwxyz"
+        return " ".join(
+            "zz" + letters[i // 26 % 26] + letters[i % 26] for i in range(n_words)
+        )
+
+    def test_extra_preamble_does_not_change_the_score(self):
+        """In the production regime (query longer than the memory — always
+        true for the UserPromptSubmit preamble), piling on more unrelated
+        preamble must NOT move the score. Under Jaccard these two differ,
+        because the union term grows with the prompt; under containment the
+        denominator is the memory, so they are identical."""
+        short = recall.containment_overlap(
+            self._filler(20) + " " + self.CORE_QUERY, self.MEMORY
+        )
+        long_ = recall.containment_overlap(
+            self._filler(200) + " " + self.CORE_QUERY, self.MEMORY
+        )
+        self.assertGreater(short, 0.0, "sanity: the memory does overlap the query")
+        self.assertAlmostEqual(
+            short,
+            long_,
+            places=6,
+            msg="prompt length changed the gate score — the union-term artifact is back",
+        )
+
+    def test_relevant_memory_survives_a_production_length_prompt(self):
+        """At the fleet default threshold (0.10, scaffold.ts), a clearly
+        relevant memory must survive a production-p50-length prompt."""
+        query = self.PREAMBLE + self.CORE_QUERY
+        self.assertGreater(
+            len(query), 400, "sanity: this is a production-shaped long prompt"
+        )
+        kept, dropped = recall._filter_by_overlap([_memory(self.MEMORY)], query, 0.10)
+        self.assertEqual(
+            len(kept), 1, "relevant memory was dropped from a long prompt"
+        )
+        self.assertEqual(dropped, 0)
+
+    def test_gate_still_drops_irrelevant_memories_on_a_long_prompt(self):
+        """The floor must still bite: a length-invariant metric must not
+        become a passthrough. An unrelated memory is still dropped."""
+        query = self.PREAMBLE + self.CORE_QUERY
+        kept, dropped = recall._filter_by_overlap(
+            [_memory("Ken prefers oat milk in his flat white.")], query, 0.10
+        )
+        self.assertEqual(len(kept), 0, "gate no longer filters — floor is dead")
+        self.assertEqual(dropped, 1)
+
+    # --- switchroom #3541 review F4: what the floor does and does NOT do ---
+    #
+    # `test_gate_still_drops_irrelevant_memories_on_a_long_prompt` above uses a
+    # memory with ZERO shared terms, so it scores 0.000 and would pass at a
+    # threshold of 0.001. It proves the gate is not a total no-op; it does not
+    # characterise the floor. These two do.
+
+    IRRELEVANT_BUT_OVERLAPPING = (
+        "The nightly digest schedule and the reaction dispatch wiring "
+        "were changed."
+    )
+
+    def test_partially_overlapping_irrelevant_memory_is_KEPT_at_the_default(self):
+        """Characterisation, not aspiration: at the 0.10 fleet default the gate
+        KEEPS an off-topic memory that merely reuses words from the prompt's
+        prior-context preamble.
+
+        This memory has nothing to do with the actual question (postgres pool
+        sizing) but shares most of its content tokens with the preamble, so it
+        scores ~0.857 — HIGHER than the genuinely relevant memory (~0.571).
+        Containment is therefore NOT monotone in relevance on a preamble-heavy
+        prompt, which is precisely why the threshold is not raised to try to
+        exclude this: any threshold that drops it drops the relevant memory
+        first. Precision here is the engine reranker's job, not the gate's.
+
+        If this test ever starts failing because the memory is now dropped,
+        the gate's selectivity changed — re-measure before accepting it.
+        """
+        query = self.PREAMBLE + self.CORE_QUERY
+        off_topic = recall.containment_overlap(query, self.IRRELEVANT_BUT_OVERLAPPING)
+        relevant = recall.containment_overlap(query, self.MEMORY)
+        self.assertGreater(
+            off_topic,
+            relevant,
+            "the relevance inversion this test documents is gone — "
+            "re-derive the threshold recommendation",
+        )
+        kept, dropped = recall._filter_by_overlap(
+            [_memory(self.IRRELEVANT_BUT_OVERLAPPING)], query, 0.10
+        )
+        self.assertEqual(len(kept), 1, "gate selectivity changed — re-measure")
+        self.assertEqual(dropped, 0)
+
+    def test_floor_drops_a_long_memory_with_only_incidental_overlap(self):
+        """Where the floor DOES bite: because the denominator is the memory's
+        own token count, a long memory sharing a single incidental term with
+        the prompt scores below 0.10 and is dropped.
+
+        This is the real content of the floor after #3541 — it removes
+        candidates with near-zero lexical relationship to the prompt, and
+        (deliberately) little else.
+        """
+        query = self.PREAMBLE + self.CORE_QUERY
+        # Exactly 1 shared content token ("docker") out of 17 -> 0.059 < 0.10
+        long_incidental = (
+            "docker buildx bake emits oci manifests whose provenance "
+            "attestations confuse older registries during garbage collection "
+            "sweeps"
+        )
+        score = recall.containment_overlap(query, long_incidental)
+        self.assertLess(score, 0.10, "fixture no longer scores below the floor")
+        self.assertGreater(score, 0.0, "sanity: there IS some incidental overlap")
+        kept, dropped = recall._filter_by_overlap(
+            [_memory(long_incidental)], query, 0.10
+        )
+        self.assertEqual(len(kept), 0, "floor is dead — nothing is being dropped")
+        self.assertEqual(dropped, 1)
+
+    # --- switchroom #3541 review F5: the |Q| < |M| regime ---
+
+    def test_short_prompt_uses_the_memory_as_denominator(self):
+        """A short prompt must NOT be scored against itself.
+
+        With the textbook overlap coefficient (`min(|Q|, |M|)`) a prompt
+        shorter than the memory flips the denominator to the QUERY, and the
+        metric silently becomes "what fraction of the prompt is in the
+        memory" — a one-word prompt then scores 1.0 against any memory
+        containing that word, re-introducing the query-length dependence
+        #3541 exists to remove. Dividing by `|M|` unconditionally has no such
+        discontinuity.
+        """
+        memory = "Postgres connection pool sizing was settled at min 5 max 100."
+        n_mem = len(recall._overlap_tokens(memory))
+        query = "docker"  # 1 content token, far shorter than the memory
+        self.assertLess(
+            len(recall._overlap_tokens(query)),
+            n_mem,
+            "sanity: this is the |Q| < |M| regime",
+        )
+        # A min()-denominator would score this 1.0 (1 shared token / |Q| = 1);
+        # dividing by the memory gives 1/2.
+        self.assertEqual(recall.containment_overlap(query, "docker image"), 0.5)
+
+        # And a short prompt overlapping the memory scores by the memory:
+        q2 = "postgres pool"
+        expected = len(
+            recall._overlap_tokens(q2) & recall._overlap_tokens(memory)
+        ) / n_mem
+        self.assertAlmostEqual(
+            recall.containment_overlap(q2, memory), expected, places=6
+        )
+
+    def test_never_stricter_than_the_jaccard_gate_it_replaces(self):
+        """Safety property vs what production runs today: because
+        `|Q u M| >= |M|`, containment >= Jaccard for every pair. At a fixed
+        threshold this gate admits a SUPERSET of what the deployed Jaccard
+        gate admits, so no memory that survives today can be dropped by this
+        change. Asserted over the fixtures rather than argued in prose.
+        """
+        query = self.PREAMBLE + self.CORE_QUERY
+        cases = [
+            self.MEMORY,
+            self.IRRELEVANT_BUT_OVERLAPPING,
+            "Ken prefers oat milk in his flat white.",
+            "docker",
+            "postgres pool sizing",
+            self.PREAMBLE,
+        ]
+        for mem in cases:
+            a = recall._overlap_tokens(query)
+            b = recall._overlap_tokens(mem)
+            jac = (len(a & b) / len(a | b)) if (a and b) else 0.0
+            score = recall.containment_overlap(query, mem)
+            self.assertGreaterEqual(
+                score + 1e-12,
+                jac,
+                f"containment scored BELOW jaccard for {mem[:40]!r} — "
+                f"this change could drop a memory production keeps",
+            )
+            # Upper bound + exact value. Review finding: the lower bound
+            # alone is unfalsifiable for ANY implementation dividing by |M|
+            # (a constant `return 1.0` satisfies it), so it survived every
+            # mutation and proved nothing. These two pin the metric.
+            self.assertLessEqual(score, 1.0, f"containment exceeded 1.0 for {mem[:40]!r}")
+            expected = (len(a & b) / len(b)) if b else 0.0
+            self.assertAlmostEqual(
+                score, expected, places=9,
+                msg=f"containment is not |Q n M| / |M| for {mem[:40]!r}",
+            )
+
+    def test_strict_subset_memory_scores_strictly_below_one(self):
+        """A memory carrying a term the query does NOT have must score < 1.0.
+
+        The companion upper bound to the superset property above: it is what
+        actually falsifies a degenerate `return 1.0` implementation.
+        """
+        query = self.PREAMBLE + self.CORE_QUERY
+        # Every token of this memory except `quokka` appears in the query.
+        mem = self.CORE_QUERY + " quokka"
+        score = recall.containment_overlap(query, mem)
+        self.assertLess(score, 1.0)
+        self.assertGreater(score, 0.0)
+        # And a fully-contained memory reaches exactly 1.0, so the < 1.0
+        # above is a property of the missing term, not a ceiling artefact.
+        self.assertAlmostEqual(
+            recall.containment_overlap(query, self.CORE_QUERY), 1.0, places=9
+        )
+
+    def test_survival_does_not_degrade_as_the_prompt_grows(self):
+        """Sweep prompt length the way production does and assert the relevant
+        memory survives at every length — the monotonic collapse is the bug."""
+        for n_filler in (0, 20, 50, 100, 200, 400):
+            query = self._filler(n_filler) + " " + self.CORE_QUERY
+            kept, _ = recall._filter_by_overlap(
+                [_memory(self.MEMORY)], query, 0.10
+            )
+            self.assertEqual(
+                len(kept),
+                1,
+                f"relevant memory dropped at {n_filler} filler words "
+                f"({len(query)} query chars)",
+            )
+
+
 class OverlapGateIntegrationTests(unittest.TestCase):
     """Switchroom #475: gate wired through main()."""
 

package/vendor/hindsight-memory/settings.json

@@ -2,7 +2,7 @@
   "hindsightApiUrl": "",
   "bankId": "claude_code",
   "bankMission": "You are a Claude Code AI assistant. Focus on technical discussions, decisions, and context relevant to the user's projects.",
-  "retainMission": "Extract technical decisions, architectural choices, user preferences, project context, and people/tool relationships. Ignore routine greetings and transient operational details.",
+  "retainMission": "Extract durable facts that will still be true and useful weeks from now: user preferences and standing rules, ongoing projects and recurring commitments, technical and architectural decisions with their rationale, and people/tool relationships. A preference revealed by a request is durable — record the preference (what the user likes, wants, or always does), not the request itself.\n\nNEVER extract:\n- Agent tool-use traces or narration of what the assistant did (e.g. \"the assistant used X to query Y\", \"ran a search\", \"sent the message\").\n- In-flight workflow/process narration (a sub-task started, paused, or is still running) — retain the outcome only once the task completes or a decision is made.\n- Operation, request, batch or session IDs, UUIDs, hashes, or error codes.\n- Hindsight's own errors, retries, backlogs, or internal state — the memory system's self-reports are not memories.\n- Restatements of the user's current request or the task in progress.\n- Transient state (unread counts, build status, what is running right now) unless the fact is explicitly dated, in which case record it as a dated observation.\n- Greetings, acknowledgements, and routine operational chatter.\n\nIf a candidate fact matches an exclusion, drop it rather than rewording it. If nothing durable remains, return an empty facts list.",
   "autoRecall": true,
   "autoRetain": true,
   "retainMode": "full-session",