boost-skill-cli 1.0.321__tar.gz → 1.0.323__tar.gz

{boost_skill_cli-1.0.321/boost_skill_cli.egg-info → boost_skill_cli-1.0.323}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: boost-skill-cli
-Version: 1.0.321
+Version: 1.0.323
 Summary: boost — Homebrew for AI coding skills
 Author: Jonathan Reyes
 License: GNU GENERAL PUBLIC LICENSE

{boost_skill_cli-1.0.321 → boost_skill_cli-1.0.323}/boost_cli/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '1.0.321'
-__version_tuple__ = version_tuple = (1, 0, 321)
+__version__ = version = '1.0.323'
+__version_tuple__ = version_tuple = (1, 0, 323)
 
 __commit_id__ = commit_id = None

{boost_skill_cli-1.0.321 → boost_skill_cli-1.0.323}/boost_cli/core/rag.py

@@ -15,13 +15,14 @@ no AI -> the BM25 order is returned as-is.
 """
 from __future__ import annotations
 
+import hashlib
 import json
 import math
 import re
 from collections import defaultdict
 from collections.abc import Sequence
 from pathlib import Path
-from typing import Dict, List, Optional, Tuple
+from typing import Dict, List, Optional, Tuple, cast
 
 try:  # TypedDict lives in typing on 3.9+, kept optional for safety
     from typing import TypedDict
@@ -33,7 +34,7 @@ from . import ai, catalog, frontmatter, gitutil, paths, registry, util
 # v2: `snip` stores a larger head of the matched chunk (not just SNIP_WIDTH
 # chars) so `retrieve` can window it onto the query terms; bump forces a
 # one-time reindex.
-INDEX_VERSION = 4
+INDEX_VERSION = 5
 ENGINE = "bm25"
 
 # Chunking defaults (documented in docs/rag-architecture.md §4).
@@ -62,10 +63,13 @@ _STOPWORDS = frozenset({
 })
 
 
-class Hit(TypedDict):  # type: ignore[misc]
+class Hit(TypedDict, total=False):  # type: ignore[misc]
     entry: dict     # the live catalog entry (name, tap, kind, skill_md, ...)
     score: float    # BM25 relevance (higher = better)
     snippet: str    # best-matching passage, for display + rerank context
+    content: str    # digest of the body, for collapsing byte-identical copies.
+                    # Optional: dense hits arrive without one and have it
+                    # attached from the BM25 index (see `_with_content`).
 
 
 # --------------------------------------------------------------- tokenizing
@@ -193,13 +197,16 @@ def _make_docs(entries: List[dict], tap_paths: Dict[str, Path]) -> List[dict]:
     docs: List[dict] = []
     for e in entries:
         body = read_body(e, tap_paths)
+        # Hashed here because the body is already in hand; doing it at query
+        # time would mean re-reading ~30k files to answer one search.
+        digest = hashlib.sha256(body.strip().encode("utf-8", "replace")).hexdigest()[:16]
         tf: Dict[str, int] = defaultdict(int)
         for tok in tokenize(surface(e) + "\n" + body):
             tf[tok] += 1
         if not tf:
             continue
         docs.append({
-            "n": e["name"], "t": e["tap"], "f": e["skill_md"],
+            "n": e["name"], "t": e["tap"], "f": e["skill_md"], "h": digest,
             "k": e.get("kind", "skill"),
             "c": 0, "l": sum(tf.values()),
             "snip": body[:SNIP_STORE].strip(),  # windowed at retrieve time
@@ -268,6 +275,7 @@ def _save(docs: List[dict], commits: Dict[str, str]) -> None:
             postings[term].append([doc_id, tf])
         total_len += d["l"]
         meta_docs.append({"n": d["n"], "t": d["t"], "f": d["f"], "k": d["k"],
+                          "h": d.get("h", ""),
                           "c": d["c"], "l": d["l"], "snip": d["snip"]})
     payload = {
         "version": INDEX_VERSION,
@@ -418,6 +426,54 @@ def entry_key(entry: dict) -> Tuple[str, str]:
     return (entry["tap"], entry["skill_md"])
 
 
+def dedupe_by_content(hits: List[Hit], limit: int) -> List[Hit]:
+    """Collapse byte-identical copies, then take ``limit``.
+
+    Measured over 77 tapped registries: 78.3% of 29,938 entries are
+    byte-identical duplicates (largest cluster 40 copies), and the
+    natural-language query set averaged 4.94 of 10 result slots consumed by a
+    repeat — worst case 9 of 10. Registries mirror each other, so N copies of one
+    rule across N taps ate N of the ten slots a user sees.
+
+    Clustering is on the **body**, never the name. 82.7% of entries share a name
+    but only 78.3% share a body, and that ~4.4% gap is real distinct content —
+    the ``admin-interface-rule`` pair fixed in #366 is two different rules under
+    one name. Clustering on content cannot make that mistake: of 14,153 distinct
+    bodies, the number of clusters spanning more than one name is zero.
+
+    Within a cluster every copy is byte-identical, so *which* one survives is
+    purely about where the user would rather install from: a ``curated`` tap wins
+    over a better-ranked uncurated one. The cluster keeps its best score either
+    way, so promoting the trusted copy never demotes the result.
+
+    A hit with no recorded content hash is always kept — two unknowns must not
+    collapse into each other, which would hide real results.
+    """
+    best: Dict[str, int] = {}          # content hash -> index into `out`
+    out: List[Hit] = []
+    for hit in hits:
+        digest = hit.get("content")
+        if not digest:
+            out.append(hit)            # unknown content is never a match
+            continue
+        seen = best.get(digest)
+        if seen is None:
+            best[digest] = len(out)
+            out.append(hit)
+            continue
+        # Same body already shown. Keep its rank, but prefer a curated source.
+        kept = out[seen]
+        if hit["entry"].get("curated") and not kept["entry"].get("curated"):
+            # cast, not a literal annotation: `Hit` is total=False for the
+            # optional `content` key, and NotRequired needs 3.11 while boost
+            # targets 3.9.
+            out[seen] = cast(Hit, {"entry": hit["entry"],
+                                   "score": kept["score"],
+                                   "snippet": kept["snippet"],
+                                   "content": digest})
+    return out[:limit]
+
+
 def retrieve(query: str, k: int = 60, kind: Optional[str] = None,
              entries: Optional[List[dict]] = None) -> List[Hit]:
     """Top-k full-content hits for ``query``. ``[]`` if the index is empty."""
@@ -430,7 +486,7 @@ def retrieve(query: str, k: int = 60, kind: Optional[str] = None,
     entries = catalog.all_entries() if entries is None else entries
     live = {entry_key(e): e for e in entries}
     docs = raw["docs"]
-    best: Dict[Tuple[str, str], Tuple[float, str]] = {}
+    best: Dict[Tuple[str, str], Tuple[float, str, str]] = {}
     for doc_id, score in _bm25(terms, raw).items():
         d = docs[doc_id]
         if kind is not None and d["k"] != kind:
@@ -440,17 +496,20 @@ def retrieve(query: str, k: int = 60, kind: Optional[str] = None,
             continue
         prev = best.get(key)
         if prev is None or score > prev[0]:
-            best[key] = (score, d["snip"])
+            best[key] = (score, d["snip"], d.get("h", ""))
     # Tie-break on the displayed name, not the identity key: the key is now
     # (tap, path), and ordering by it would silently reshuffle equal-scoring
     # results relative to every previous release.
     ranked = sorted(best.items(),
                     key=lambda kv: (-kv[1][0], live[kv[0]]["name"], kv[0]))
+    # Over-fetch, then collapse byte-identical copies before taking k — the
+    # duplicates have to be removed from the pool, not from the visible page,
+    # or they still consume the slots they were removed from.
     hits: List[Hit] = [
-        {"entry": live[key], "score": score,
+        {"entry": live[key], "score": score, "content": digest,
          "snippet": _passage(snip, terms)}  # type: ignore[typeddict-item]
-        for key, (score, snip) in ranked[:k]]
-    return hits
+        for key, (score, snip, digest) in ranked]
+    return dedupe_by_content(hits, k)
 
 
 def rerank(query: str, hits: List[Hit], limit: int = 10) -> Tuple[List[Hit], str]:
@@ -516,6 +575,39 @@ def rrf_fuse(rankings: List[List[Hit]], limit: int = 60) -> List[Hit]:
     return fused
 
 
+def content_hashes() -> Dict[Tuple[str, str], str]:
+    """``entry_key`` -> content hash, read from the BM25 index.
+
+    The dense store does not carry the hash and does not need to: BM25 indexes
+    the same entries and auto-builds on first search, so one map serves every
+    engine. Keeping it in one place also means the two engines can never
+    disagree about which copies are identical.
+    """
+    raw = _load_raw()
+    if not raw:
+        return {}
+    return {(d["t"], d["f"]): d["h"]
+            for d in raw.get("docs", []) if d.get("h")}
+
+
+def _with_content(hits: List[Hit]) -> List[Hit]:
+    """Attach content hashes to hits that lack them (dense, and fused output)."""
+    if all(h.get("content") for h in hits):
+        return hits
+    hashes = content_hashes()
+    if not hashes:
+        return hits
+    filled: List[Hit] = []
+    for h in hits:
+        if h.get("content"):
+            filled.append(h)
+            continue
+        filled.append(cast(Hit, {
+            "entry": h["entry"], "score": h["score"], "snippet": h["snippet"],
+            "content": hashes.get(entry_key(h["entry"]), "")}))
+    return filled
+
+
 def retrieve_any(query: str, k: int = 60, kind: Optional[str] = None,
                  entries: Optional[List[dict]] = None
                  ) -> Tuple[Optional[List[Hit]], str]:
@@ -551,13 +643,18 @@ def retrieve_any(query: str, k: int = 60, kind: Optional[str] = None,
     # counts as a ranking to fuse.
     bm25_hits = retrieve(query, k=pool, kind=kind,
                          entries=entries) if ready() else None
+    # Dedupe at this seam as well as inside `retrieve`, because dense hits carry
+    # no hash of their own and fusion can reintroduce a duplicate that BM25
+    # already dropped — the copies are distinct `(tap, path)` keys, so RRF has
+    # no reason to treat them as one.
     if dense_hits and bm25_hits:
-        return (rrf_fuse([bm25_hits, dense_hits], limit=k),
+        fused = rrf_fuse([bm25_hits, dense_hits], limit=max(k, RRF_K))
+        return (dedupe_by_content(_with_content(fused), k),
                 "hybrid RRF (BM25 + dense)")
     if dense_hits:
-        return dense_hits[:k], "dense vectors"
+        return dedupe_by_content(_with_content(dense_hits), k), "dense vectors"
     if bm25_hits is not None:
-        return bm25_hits[:k], "BM25 full-content"
+        return bm25_hits[:k], "BM25 full-content"     # already deduped
     return None, ""
 
 

{boost_skill_cli-1.0.321 → boost_skill_cli-1.0.323/boost_skill_cli.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: boost-skill-cli
-Version: 1.0.321
+Version: 1.0.323
 Summary: boost — Homebrew for AI coding skills
 Author: Jonathan Reyes
 License: GNU GENERAL PUBLIC LICENSE

{boost_skill_cli-1.0.321 → boost_skill_cli-1.0.323}/boost_skill_cli.egg-info/SOURCES.txt

@@ -483,6 +483,7 @@ tests/unit/test_codeql_analysis_key.py
 tests/unit/test_command_reference_fresh.py
 tests/unit/test_community_health_files.py
 tests/unit/test_config.py
+tests/unit/test_content_dedupe.py
 tests/unit/test_dense.py
 tests/unit/test_dense_fallback.py
 tests/unit/test_dense_fix_hint.py

{boost_skill_cli-1.0.321 → boost_skill_cli-1.0.323}/docs/roadmap/items/near-duplicate-items-eat-the-result-slots.md

@@ -9,7 +9,7 @@ impact: High
 wow: 4
 note: 56.3% of entries share a name
 order: 71
-owner: loop/entry-key-collision
+owner: loop/content-hash-dedup
 pr:
 title: Near-duplicate items consume the top-10, and it gets worse with every tap
 ---
@@ -91,3 +91,71 @@ at one path — and were corrected.
 prior, and the 83-tap re-measurement of the fusion bound are all untouched. Note the two are
 independent: dedup <em>merges</em> copies that are genuinely the same, whereas this fix stops copies
 that were never the same from being merged by accident.
+
+<b>The curve this card asked for.</b> The note above measured 0 duplicate slots over the pinned
+6-tap corpus and said, correctly, that absence at 6 taps says nothing about 83. Re-measured over 77
+tapped registries sampled across the shipped list, with the same 50 natural-language queries,
+counting how many of the ten slots a user sees are second-or-later copies of a name already in that
+same result list:
+
+<code>6 taps</code>: 1,122 entries, 0.0% sharing a name, <b>0.00</b> duplicate slots/query &middot;
+<code>12</code>: 14,194 entries, 77.8%, <b>4.28</b> &middot;
+<code>25</code>: 16,633, 77.9%, 4.12 &middot;
+<code>40</code>: 17,470, 74.7%, 3.88 &middot;
+<code>60</code>: 29,137, 83.9%, 5.08 &middot;
+<code>77</code>: 29,938 entries, 82.7% sharing a name, <b>4.94</b> duplicate slots/query.
+
+So at realistic corpus size <b>roughly half the ten slots a user sees are repeat copies of a name
+already in the list</b>, worst observed 9 of 10 (&ldquo;our full-text queries have got slow as the
+index grew&rdquo;). The card's thesis holds, and more sharply than its own 56.3% figure suggested.
+
+<b>But the driver is not tap count.</b> Entries jump 1,122 &rarr; 14,194 between 6 and 12 taps, so
+the sample is not uniform: a single awesome-list-style rule registry contributes more entries than
+the entire 6-tap eval corpus. Duplicate pressure is a step function of <em>which</em> registries are
+tapped, not a smooth function of how many &mdash; it appears the moment one duplicate-heavy registry
+is added and then stays flat at ~4-5 slots per query from 12 taps to 77. Any dedup benefit should be
+reported against a stated tap set for that reason, and a user who taps only curated skill registries
+may never see the problem at all.
+
+<b>What this does not settle.</b> The measurement counts repeated <em>names</em>, which is the
+symptom the card names, not content-identical bodies &mdash; two rules that share a name and
+genuinely differ (the <code>admin-interface-rule</code> pair fixed in <code>#366</code>) are counted
+here as duplicate slots but must <em>not</em> be merged. That is precisely why the card asks for
+clustering on content hash rather than on name, and this curve is an upper bound on what dedup could
+reclaim, not a target.
+
+<b>Shipped: content-hash dedup, measured 4.94 &rarr; 0.60 duplicate slots per query</b> over the
+77-tap corpus (worst case 9 of 10 &rarr; 4). The residual 0.60 is correct rather than leftover: those
+are entries sharing a <em>name</em> whose bodies genuinely differ, which must stay separate.
+
+Two measurements settled the design the card left open (&ldquo;content hash <em>or</em> canonical
+name&rdquo;). Of 29,938 entries, <b>78.3% are byte-identical duplicates</b> &mdash; 14,153 distinct
+bodies, largest cluster <b>40 copies</b> &mdash; while 82.7% share a name. That ~4.4% gap is real
+distinct content, so name clustering would merge exactly the pairs <code>#366</code> proved must not
+be merged. And clustering on content cannot make the opposite mistake here: the number of content
+clusters spanning more than one name is <b>0</b>, so collapsing by body never merges two
+differently-named items. Content hashing is strictly the safer of the two options the card offered,
+which was not obvious before measuring.
+
+The quality prior is <code>curated</code>: inside a cluster every copy is byte-identical, so the one
+worth surfacing is the one from a tap the user marked trusted. The cluster keeps its best score when
+that swap happens, so promoting the trusted copy never demotes the result.
+
+Mechanically, the body hash is computed in <code>_make_docs</code> where the body is already read
+&mdash; doing it at query time would mean re-reading ~30k files to answer one search &mdash; and
+persisted per document (<code>INDEX_VERSION</code> 4 &rarr; 5). Dense hits carry no hash of their own
+and do not need a second schema: <code>content_hashes()</code> serves one map to every engine from
+the BM25 index, so the engines can never disagree about which copies are identical. Dedup runs inside
+<code>retrieve</code> <em>and</em> at the <code>retrieve_any</code> seam, because fusion can
+reintroduce a copy BM25 already dropped &mdash; the copies are distinct <code>(tap, path)</code>
+keys, so RRF has no reason to treat them as one. Collapsing happens before <code>k</code> is applied,
+or the duplicates would still consume the slots they were removed from.
+
+Cost on the pinned eval corpus: <b>none</b> &mdash; 0.978 / 0.791 / 0.854 / 0.882, identical to
+before, all four floors passing. That corpus has little content duplication, which is exactly why the
+83-tap measurement had to exist.
+
+<b>Still open:</b> a richer quality prior than <code>curated</code> (the card names stars, recency,
+maintenance; only <code>curated</code> is available on an entry today), and
+<code>core/typosquat.py</code>'s confusion machinery for near-identical rather than byte-identical
+bodies. Both are refinements &mdash; the byte-identical case is 78.3% of the problem.

{boost_skill_cli-1.0.321 → boost_skill_cli-1.0.323}/docs/roadmap.html

@@ -3440,7 +3440,75 @@ at one path — and were corrected.
 <b>Still open:</b> the ranking half. Clustering on content hash, picking a winner with a quality
 prior, and the 83-tap re-measurement of the fusion bound are all untouched. Note the two are
 independent: dedup <em>merges</em> copies that are genuinely the same, whereas this fix stops copies
-that were never the same from being merged by accident.</p>
+that were never the same from being merged by accident.
+
+<b>The curve this card asked for.</b> The note above measured 0 duplicate slots over the pinned
+6-tap corpus and said, correctly, that absence at 6 taps says nothing about 83. Re-measured over 77
+tapped registries sampled across the shipped list, with the same 50 natural-language queries,
+counting how many of the ten slots a user sees are second-or-later copies of a name already in that
+same result list:
+
+<code>6 taps</code>: 1,122 entries, 0.0% sharing a name, <b>0.00</b> duplicate slots/query &middot;
+<code>12</code>: 14,194 entries, 77.8%, <b>4.28</b> &middot;
+<code>25</code>: 16,633, 77.9%, 4.12 &middot;
+<code>40</code>: 17,470, 74.7%, 3.88 &middot;
+<code>60</code>: 29,137, 83.9%, 5.08 &middot;
+<code>77</code>: 29,938 entries, 82.7% sharing a name, <b>4.94</b> duplicate slots/query.
+
+So at realistic corpus size <b>roughly half the ten slots a user sees are repeat copies of a name
+already in the list</b>, worst observed 9 of 10 (&ldquo;our full-text queries have got slow as the
+index grew&rdquo;). The card's thesis holds, and more sharply than its own 56.3% figure suggested.
+
+<b>But the driver is not tap count.</b> Entries jump 1,122 &rarr; 14,194 between 6 and 12 taps, so
+the sample is not uniform: a single awesome-list-style rule registry contributes more entries than
+the entire 6-tap eval corpus. Duplicate pressure is a step function of <em>which</em> registries are
+tapped, not a smooth function of how many &mdash; it appears the moment one duplicate-heavy registry
+is added and then stays flat at ~4-5 slots per query from 12 taps to 77. Any dedup benefit should be
+reported against a stated tap set for that reason, and a user who taps only curated skill registries
+may never see the problem at all.
+
+<b>What this does not settle.</b> The measurement counts repeated <em>names</em>, which is the
+symptom the card names, not content-identical bodies &mdash; two rules that share a name and
+genuinely differ (the <code>admin-interface-rule</code> pair fixed in <code>#366</code>) are counted
+here as duplicate slots but must <em>not</em> be merged. That is precisely why the card asks for
+clustering on content hash rather than on name, and this curve is an upper bound on what dedup could
+reclaim, not a target.
+
+<b>Shipped: content-hash dedup, measured 4.94 &rarr; 0.60 duplicate slots per query</b> over the
+77-tap corpus (worst case 9 of 10 &rarr; 4). The residual 0.60 is correct rather than leftover: those
+are entries sharing a <em>name</em> whose bodies genuinely differ, which must stay separate.
+
+Two measurements settled the design the card left open (&ldquo;content hash <em>or</em> canonical
+name&rdquo;). Of 29,938 entries, <b>78.3% are byte-identical duplicates</b> &mdash; 14,153 distinct
+bodies, largest cluster <b>40 copies</b> &mdash; while 82.7% share a name. That ~4.4% gap is real
+distinct content, so name clustering would merge exactly the pairs <code>#366</code> proved must not
+be merged. And clustering on content cannot make the opposite mistake here: the number of content
+clusters spanning more than one name is <b>0</b>, so collapsing by body never merges two
+differently-named items. Content hashing is strictly the safer of the two options the card offered,
+which was not obvious before measuring.
+
+The quality prior is <code>curated</code>: inside a cluster every copy is byte-identical, so the one
+worth surfacing is the one from a tap the user marked trusted. The cluster keeps its best score when
+that swap happens, so promoting the trusted copy never demotes the result.
+
+Mechanically, the body hash is computed in <code>_make_docs</code> where the body is already read
+&mdash; doing it at query time would mean re-reading ~30k files to answer one search &mdash; and
+persisted per document (<code>INDEX_VERSION</code> 4 &rarr; 5). Dense hits carry no hash of their own
+and do not need a second schema: <code>content_hashes()</code> serves one map to every engine from
+the BM25 index, so the engines can never disagree about which copies are identical. Dedup runs inside
+<code>retrieve</code> <em>and</em> at the <code>retrieve_any</code> seam, because fusion can
+reintroduce a copy BM25 already dropped &mdash; the copies are distinct <code>(tap, path)</code>
+keys, so RRF has no reason to treat them as one. Collapsing happens before <code>k</code> is applied,
+or the duplicates would still consume the slots they were removed from.
+
+Cost on the pinned eval corpus: <b>none</b> &mdash; 0.978 / 0.791 / 0.854 / 0.882, identical to
+before, all four floors passing. That corpus has little content duplication, which is exactly why the
+83-tap measurement had to exist.
+
+<b>Still open:</b> a richer quality prior than <code>curated</code> (the card names stars, recency,
+maintenance; only <code>curated</code> is available on an entry today), and
+<code>core/typosquat.py</code>'s confusion machinery for near-identical rather than byte-identical
+bodies. Both are refinements &mdash; the byte-identical case is 78.3% of the problem.</p>
         <div class="meta">
           <span class="m">Complexity <b>M</b></span>
           <span class="m hi">Impact <b>High</b></span>

boost_skill_cli-1.0.323/tests/unit/test_content_dedupe.py

@@ -0,0 +1,144 @@
+"""Collapsing byte-identical copies before they reach the ten slots a user sees.
+
+Measured over 77 tapped registries (29,938 entries): **78.3% are byte-identical
+duplicates**, largest cluster 40 copies, and the natural-language query set
+averages **4.94 of 10 result slots** consumed by a repeat of a name already in
+the list — worst case 9 of 10.
+
+Two measurements decide the design:
+
+* **Cluster on content, never on name.** 82.7% of entries share a name but only
+  78.3% share a body. That ~4.4% gap is real distinct content — the
+  `admin-interface-rule` pair fixed in #366 is two different rules under one
+  name — and name-based dedup would merge exactly those.
+* **Content clustering cannot over-merge here.** Of 14,153 distinct bodies, the
+  number of clusters spanning more than one name is **0**. So collapsing by body
+  never merges two differently-named items; it is strictly safer than the
+  name-based alternative the card also offered.
+
+The quality prior is `curated`: within a cluster every copy is byte-identical,
+so the one worth surfacing is the one from a tap the user marked trusted.
+"""
+from __future__ import annotations
+
+from boost_cli.core import rag
+
+
+def _hit(name, tap, score, content, curated=False, skill_md=None):
+    return {"entry": {"name": name, "tap": tap, "curated": curated,
+                      "skill_md": skill_md or ("%s/SKILL.md" % name),
+                      "kind": "skill", "description": ""},
+            "score": score, "snippet": "", "content": content}
+
+
+class TestCollapsesIdenticalBodies:
+    def test_two_byte_identical_copies_become_one(self):
+        hits = [_hit("rule", "a/x", 3.0, "same"), _hit("rule", "b/y", 2.0, "same")]
+        out = rag.dedupe_by_content(hits, limit=10)
+        assert len(out) == 1
+
+    def test_the_best_ranked_copy_is_the_one_kept(self):
+        hits = [_hit("rule", "a/x", 3.0, "same"), _hit("rule", "b/y", 2.0, "same")]
+        assert rag.dedupe_by_content(hits, limit=10)[0]["entry"]["tap"] == "a/x"
+
+    def test_a_cluster_of_forty_becomes_one(self):
+        # The largest real cluster measured over 77 taps.
+        hits = [_hit("rule", "t%d/r" % i, 40.0 - i, "same") for i in range(40)]
+        assert len(rag.dedupe_by_content(hits, limit=10)) == 1
+
+    def test_order_of_surviving_hits_is_preserved(self):
+        hits = [_hit("a", "t/1", 5.0, "A"), _hit("b", "t/2", 4.0, "B"),
+                _hit("a2", "t/3", 3.0, "A")]          # dupe of the first
+        out = rag.dedupe_by_content(hits, limit=10)
+        assert [h["entry"]["name"] for h in out] == ["a", "b"]
+
+
+class TestNeverMergesDistinctContent:
+    """The failure that would matter: two different rules collapsed into one."""
+
+    def test_same_name_different_bodies_stay_two_results(self):
+        # The #366 pair: one tap, one name, two genuinely different rules.
+        hits = [_hit("admin-interface-rule", "z/rules", 3.0, "express mongodb",
+                     skill_md="a/express-mongodb/admin-interface-rule.mdc"),
+                _hit("admin-interface-rule", "z/rules", 2.0, "mern fullstack",
+                     skill_md="a/fullstack-mern/admin-interface-rule.mdc")]
+        out = rag.dedupe_by_content(hits, limit=10)
+        assert len(out) == 2, "distinct bodies were merged on a shared name"
+
+    def test_missing_content_is_never_treated_as_a_match(self):
+        # Two hits with no recorded hash must not collapse into each other just
+        # because both are unknown — that would silently hide real results.
+        hits = [_hit("a", "t/1", 3.0, None), _hit("b", "t/2", 2.0, None)]
+        assert len(rag.dedupe_by_content(hits, limit=10)) == 2
+
+
+class TestQualityPrior:
+    """Every copy in a cluster is byte-identical, so surface the trusted one."""
+
+    def test_a_curated_copy_wins_over_a_better_ranked_uncurated_one(self):
+        hits = [_hit("rule", "rando/x", 3.0, "same"),
+                _hit("rule", "trusted/y", 2.0, "same", curated=True)]
+        kept = rag.dedupe_by_content(hits, limit=10)[0]["entry"]
+        assert kept["tap"] == "trusted/y"
+
+    def test_the_kept_hit_keeps_the_best_score_in_its_cluster(self):
+        # Swapping which copy is shown must not demote the cluster's ranking.
+        hits = [_hit("rule", "rando/x", 3.0, "same"),
+                _hit("rule", "trusted/y", 2.0, "same", curated=True)]
+        assert rag.dedupe_by_content(hits, limit=10)[0]["score"] == 3.0
+
+    def test_two_curated_copies_fall_back_to_rank(self):
+        hits = [_hit("rule", "a/x", 3.0, "same", curated=True),
+                _hit("rule", "b/y", 2.0, "same", curated=True)]
+        assert rag.dedupe_by_content(hits, limit=10)[0]["entry"]["tap"] == "a/x"
+
+
+class TestLimit:
+    def test_limit_is_applied_after_collapsing_not_before(self):
+        # The whole point: duplicates must not consume the slots. Ten copies of
+        # one body followed by three distinct items should yield four results,
+        # not one.
+        hits = ([_hit("dup", "t%d/r" % i, 20.0 - i, "same") for i in range(10)]
+                + [_hit("x", "t/a", 5.0, "X"), _hit("y", "t/b", 4.0, "Y"),
+                   _hit("z", "t/c", 3.0, "Z")])
+        out = rag.dedupe_by_content(hits, limit=4)
+        assert [h["entry"]["name"] for h in out] == ["dup", "x", "y", "z"]
+
+    def test_empty_input_is_empty_output(self):
+        assert rag.dedupe_by_content([], limit=10) == []
+
+
+class TestHashesReachEveryEngine:
+    """Dense hits carry no hash of their own; one map serves all engines."""
+
+    def test_hashes_are_read_from_the_bm25_index(self, sandbox):
+        rag._save([{"n": "a", "t": "x/y", "f": "a/SKILL.md", "h": "deadbeef",
+                    "k": "skill", "c": 0, "l": 1, "snip": "s",
+                    "tf": {"z": 1}}], {})
+        rag._CACHE.clear()
+        assert rag.content_hashes() == {("x/y", "a/SKILL.md"): "deadbeef"}
+
+    def test_a_dense_hit_gets_its_hash_attached(self, sandbox):
+        rag._save([{"n": "a", "t": "x/y", "f": "a/SKILL.md", "h": "deadbeef",
+                    "k": "skill", "c": 0, "l": 1, "snip": "s",
+                    "tf": {"z": 1}}], {})
+        rag._CACHE.clear()
+        bare = [{"entry": {"name": "a", "tap": "x/y",
+                           "skill_md": "a/SKILL.md"},
+                 "score": 1.0, "snippet": ""}]
+        assert rag._with_content(bare)[0]["content"] == "deadbeef"
+
+    def test_an_existing_hash_is_not_overwritten(self, sandbox):
+        hits = [{"entry": {"name": "a", "tap": "x/y", "skill_md": "a/SKILL.md"},
+                 "score": 1.0, "snippet": "", "content": "mine"}]
+        assert rag._with_content(hits)[0]["content"] == "mine"
+
+    def test_no_index_degrades_to_leaving_hits_alone(self, sandbox):
+        # No BM25 index yet: dedupe must no-op rather than collapse everything
+        # into one result on a shared empty hash.
+        bare = [{"entry": {"name": "a", "tap": "x/y", "skill_md": "a.md"},
+                 "score": 2.0, "snippet": ""},
+                {"entry": {"name": "b", "tap": "x/y", "skill_md": "b.md"},
+                 "score": 1.0, "snippet": ""}]
+        out = rag.dedupe_by_content(rag._with_content(bare), limit=10)
+        assert len(out) == 2

{boost_skill_cli-1.0.321 → boost_skill_cli-1.0.323}/tests/unit/test_rag.py

@@ -741,9 +741,11 @@ class TestSavePayload:
         assert sorted(raw["postings"]["foo"]) == [[0, 1], [1, 2]]
         assert raw["postings"]["bar"] == [[0, 1]]
         assert raw["docs"][0] == {"n": "a", "t": "x/y", "f": "a/SKILL.md",
-                                  "k": "skill", "c": 0, "l": 2, "snip": "sa"}
+                                  "h": "", "k": "skill", "c": 0, "l": 2,
+                                  "snip": "sa"}
         assert raw["docs"][1] == {"n": "b", "t": "x/y", "f": "b.mdc",
-                                  "k": "rule", "c": 3, "l": 4, "snip": "sb"}
+                                  "h": "", "k": "rule", "c": 3, "l": 4,
+                                  "snip": "sb"}
 
     def test_empty_docs_avg_len_zero(self, sandbox):
         rag._save([], {})