boost-skill-cli 1.2.72__tar.gz → 1.2.74__tar.gz

{boost_skill_cli-1.2.72/boost_skill_cli.egg-info → boost_skill_cli-1.2.74}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: boost-skill-cli
-Version: 1.2.72
+Version: 1.2.74
 Summary: boost — Homebrew for AI coding skills
 Author: Jonathan Reyes
 License-Expression: Apache-2.0

{boost_skill_cli-1.2.72 → boost_skill_cli-1.2.74}/boost_cli/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '1.2.72'
-__version_tuple__ = version_tuple = (1, 2, 72)
+__version__ = version = '1.2.74'
+__version_tuple__ = version_tuple = (1, 2, 74)
 
 __commit_id__ = commit_id = None

{boost_skill_cli-1.2.72 → boost_skill_cli-1.2.74}/boost_cli/commands/discovery.py

@@ -96,6 +96,12 @@ def cmd_search(argv):
                    help="rerank the top hits with Claude")
     p.add_argument("--limit", type=util.positive_int, default=15,
                    help="max results (default 15)")
+    p.add_argument("--collapse-near-duplicates", action="store_true",
+                   dest="collapse_dupes",
+                   help="collapse near-identical entries (translations, "
+                        "paraphrases) via dense embeddings — opt-in, needs a "
+                        "built dense index; threshold not yet validated "
+                        "against a real corpus, see the roadmap")
     p.add_argument("--json", action="store_true", dest="as_json",
                    help="machine-readable output")
     args = p.parse_args(argv)
@@ -127,7 +133,9 @@ def cmd_search(argv):
         # retrieve_any, not retrieve: this picks the dense backend when one is
         # built and floors to BM25 otherwise, so the CLI and the MCP server
         # answer from the same engine instead of the CLI being BM25-only.
-        hits, engine = rag.retrieve_any(query, k=max(60, args.limit * 4))
+        hits, engine = rag.retrieve_any(
+            query, k=max(60, args.limit * 4),
+            collapse_near_duplicates=args.collapse_dupes)
         scored = [(h["entry"], h["score"]) for h in (hits or [])]
     else:
         scored = catalog.search(query)

{boost_skill_cli-1.2.72 → boost_skill_cli-1.2.74}/boost_cli/core/dense.py

@@ -1561,6 +1561,51 @@ def _knn(con: sqlite3.Connection, qblob: bytes,
     return _expand(con, ranked, pool)
 
 
+def entry_vectors(entry_keys: list[tuple[str, str]],
+                  ) -> dict[tuple[str, str], bytes]:
+    """Raw float32 embedding for each entry's first chunk, keyed by ``(tap, path)``.
+
+    Feeds `rag.collapse_near_duplicate_hits`, whose clustering needs one
+    representative vector per *entry* rather than per chunk. The lowest-``cix``
+    chunk is that representative: `rag.chunk` always emits the name+description
+    chunk first, which is enough signal to place a translation or a paraphrase
+    near its original without reading the rest of the body.
+
+    Only a quantized store answers this cheaply: `vec_raw` is a plain
+    rowid-keyed table, so looking vectors up by id is an index probe (see
+    `_knn`'s comment on why the same lookup against a `vec0` table plans as a
+    full scan instead). A legacy float32 store returns ``{}``, and near-dup
+    collapsing degrades to a no-op there — the same way every other feature on
+    that path degrades, rather than paying a full-scan cost per entry.
+    """
+    if not entry_keys:
+        return {}
+    con = _connect()
+    if con is None:
+        return {}
+    try:
+        if not quantized(con):
+            return {}
+        col = _vid_col(con)
+        vids: dict[tuple[str, str], int] = {}
+        for tap, path in entry_keys:
+            row = con.execute(
+                "SELECT %s FROM chunks WHERE tap = ? AND path = ? "  # noqa: S608  col is a literal from _vid_col; tap/path are bound
+                "ORDER BY cix LIMIT 1" % col, (tap, path)).fetchone()
+            if row is not None and row[0] is not None:
+                vids[(tap, path)] = row[0]
+        if not vids:
+            return {}
+        ids = sorted(set(vids.values()))
+        rows = con.execute(
+            "SELECT id, embedding FROM vec_raw WHERE id IN (%s)"  # noqa: S608  interpolates only `?` placeholders; ids are bound
+            % ",".join("?" * len(ids)), ids).fetchall()
+        blobs = dict(rows)
+        return {key: blobs[vid] for key, vid in vids.items() if vid in blobs}
+    finally:
+        con.close()
+
+
 def retrieve(query: str, k: int = 60, kind: str | None = None,
              entries: list[dict] | None = None) -> list[Hit] | None:
     """Top-k dense hits for ``query``, or None to signal 'fall back to BM25'."""

{boost_skill_cli-1.2.72 → boost_skill_cli-1.2.74}/boost_cli/core/rag.py

@@ -17,6 +17,7 @@ no AI -> the BM25 order is returned as-is.
 """
 from __future__ import annotations
 
+import array
 import contextlib
 import hashlib
 import json
@@ -773,6 +774,91 @@ def dedupe_by_content(hits: list[Hit], limit: int) -> list[Hit]:
     return out[:limit]
 
 
+#: Cosine-similarity floor for two entries to collapse as near-duplicates.
+#:
+#: Deliberately conservative and deliberately unproven. `dedupe_by_content`
+#: above shipped only after one count settled its safety: of 14,153 distinct
+#: bodies, the clusters spanning more than one *name* numbered zero, so
+#: collapsing on content hash could not merge two different skills. No
+#: equivalent count exists yet for near-identical clustering — the measurement
+#: the roadmap card asks for is "over a real corpus, at the chosen threshold,
+#: count clusters spanning more than one meaning" and nobody has run it. Until
+#: someone does, `collapse_near_duplicate_hits` stays reachable only opt-in
+#: (see `retrieve_any`'s `collapse_near_duplicates` parameter), so the default
+#: search path is unaffected by a threshold nobody has validated.
+NEAR_DUPLICATE_THRESHOLD = 0.97
+
+
+def _decode_vector(blob: bytes) -> array.array:
+    """A stored float32 embedding blob as a sequence of floats."""
+    vec = array.array("f")
+    vec.frombytes(blob)
+    return vec
+
+
+def _cosine(a: array.array, b: array.array) -> float:
+    """Cosine similarity of two float32 vectors, 0.0 for any comparison that
+    cannot be trusted: a dimension mismatch (different embedding models) or
+    either side being the zero vector. An ambiguous comparison must never read
+    as a match, the same rule a missing content hash follows above.
+    """
+    if not a or not b or len(a) != len(b):
+        return 0.0
+    dot = sum(x * y for x, y in zip(a, b, strict=True))
+    norm_a = math.sqrt(sum(x * x for x in a))
+    norm_b = math.sqrt(sum(x * x for x in b))
+    if 0.0 in (norm_a, norm_b):
+        return 0.0
+    return dot / (norm_a * norm_b)
+
+
+def collapse_near_duplicate_hits(hits: list[Hit],
+                                  vectors: dict[tuple[str, str], bytes],
+                                  limit: int,
+                                  threshold: float = NEAR_DUPLICATE_THRESHOLD,
+                                  ) -> list[Hit]:
+    """Collapse entries whose embeddings are near-identical, then take ``limit``.
+
+    Runs on the output of :func:`dedupe_by_content`, on the same contract —
+    keep the earliest rank slot, promote a better source into it — for the
+    residual that survives byte-identical dedup: the same skill translated or
+    paraphrased across mirrors. Observed on a real 466-tap install: the query
+    ``exa search`` returned ten rows that were all one skill in Japanese,
+    Chinese, and five English variants, none sharing a body digest, so
+    ``dedupe_by_content`` correctly kept every one of them apart.
+
+    ``vectors`` maps ``entry_key(hit["entry"])`` to a raw float32 embedding
+    blob, looked up by the caller (:func:`boost_cli.core.dense.entry_vectors`)
+    rather than fetched here — this stays pure so it is cheap to test with
+    synthetic vectors, and clustering ``hits`` is O(n * clusters), never
+    O(n^2) against the whole store.
+
+    A hit with no vector — no dense store built, or the entry never got one —
+    is never merged into anything: two unknowns must not collapse into each
+    other, the same rule :func:`dedupe_by_content` follows for a missing
+    content hash.
+    """
+    out: list[Hit] = []
+    reps: list[array.array | None] = []   # parallel to `out`
+    for hit in hits:
+        blob = vectors.get(entry_key(hit["entry"]))
+        vec = _decode_vector(blob) if blob else None
+        merge_at = None
+        if vec is not None:
+            for i, rep in enumerate(reps):
+                if rep is not None and _cosine(vec, rep) >= threshold:
+                    merge_at = i
+                    break
+        if merge_at is None:
+            out.append(hit)
+            reps.append(vec)
+            continue
+        kept = out[merge_at]
+        if source_rank(hit["entry"]) < source_rank(kept["entry"]):
+            out[merge_at] = cast(Hit, {**kept, "entry": hit["entry"]})
+    return out[:limit]
+
+
 def _windowed(hits: list[Hit], terms: Sequence[str]) -> list[Hit]:
     """Window each hit's raw stored snip onto the query terms.
 
@@ -1087,7 +1173,8 @@ def _with_content(hits: list[Hit]) -> list[Hit]:
 
 
 def retrieve_any(query: str, k: int = 60, kind: str | None = None,
-                 entries: list[dict] | None = None
+                 entries: list[dict] | None = None,
+                 collapse_near_duplicates: bool = False,
                  ) -> tuple[list[Hit] | None, str]:
     """Fuse BM25 and dense when both are available; floor to whichever is.
 
@@ -1107,6 +1194,12 @@ def retrieve_any(query: str, k: int = 60, kind: str | None = None,
     Both engines are over-fetched to ``RRF_K`` before fusing. Fusing only the
     top-k of each would discard exactly the candidates the other engine was
     going to promote, which is most of the value.
+
+    ``collapse_near_duplicates`` runs :func:`collapse_near_duplicate_hits` over
+    the result, on top of the byte-identical dedup every branch below already
+    does — off by default (see :data:`NEAR_DUPLICATE_THRESHOLD` for why) and a
+    no-op whenever no dense store is built, since that is the only source of
+    the embeddings the collapse needs.
     """
     from . import dense
     pool = max(k, RRF_K)
@@ -1127,13 +1220,20 @@ def retrieve_any(query: str, k: int = 60, kind: str | None = None,
     # no reason to treat them as one.
     if dense_hits and bm25_hits:
         fused = rrf_fuse([bm25_hits, dense_hits], limit=max(k, RRF_K))
-        return (dedupe_by_content(_with_content(fused), k),
-                "hybrid RRF (BM25 + dense)")
-    if dense_hits:
-        return dedupe_by_content(_with_content(dense_hits), k), "dense vectors"
-    if bm25_hits is not None:
-        return bm25_hits[:k], "BM25 full-content"     # already deduped
-    return None, ""
+        hits, engine = dedupe_by_content(_with_content(fused), k), \
+            "hybrid RRF (BM25 + dense)"
+    elif dense_hits:
+        hits, engine = dedupe_by_content(_with_content(dense_hits), k), \
+            "dense vectors"
+    elif bm25_hits is not None:
+        hits, engine = bm25_hits[:k], "BM25 full-content"     # already deduped
+    else:
+        return None, ""
+    if collapse_near_duplicates and hits and dense.ready():
+        vectors = dense.entry_vectors([entry_key(h["entry"]) for h in hits])
+        if vectors:
+            hits = collapse_near_duplicate_hits(hits, vectors, k)
+    return hits, engine
 
 
 def search(query: str, limit: int = 10, kind: str | None = None,

{boost_skill_cli-1.2.72 → boost_skill_cli-1.2.74/boost_skill_cli.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: boost-skill-cli
-Version: 1.2.72
+Version: 1.2.74
 Summary: boost — Homebrew for AI coding skills
 Author: Jonathan Reyes
 License-Expression: Apache-2.0

{boost_skill_cli-1.2.72 → boost_skill_cli-1.2.74}/boost_skill_cli.egg-info/SOURCES.txt

@@ -657,6 +657,7 @@ scripts/eval_stats_summary.py
 scripts/eval_tools.py
 scripts/import_budget.py
 scripts/lock_toolchain.py
+scripts/measure_near_duplicate_bound.py
 scripts/measure_registry.py
 scripts/mutation_gate.py
 scripts/mutation_shards.py
@@ -803,6 +804,7 @@ tests/unit/test_dense.py
 tests/unit/test_dense_boilerplate_cluster.py
 tests/unit/test_dense_embed_dedupe.py
 tests/unit/test_dense_entry_reuse.py
+tests/unit/test_dense_entry_vectors.py
 tests/unit/test_dense_fallback.py
 tests/unit/test_dense_fix_hint.py
 tests/unit/test_dense_progress.py
@@ -869,6 +871,7 @@ tests/unit/test_mutation_hardening.py
 tests/unit/test_mutation_shards.py
 tests/unit/test_mutation_subfile_shards.py
 tests/unit/test_mutmut_source_paths.py
+tests/unit/test_near_duplicate_collapse.py
 tests/unit/test_nethttp.py
 tests/unit/test_normalize_sdist.py
 tests/unit/test_output.py
@@ -898,6 +901,7 @@ tests/unit/test_release_preflight.py
 tests/unit/test_required_checks.py
 tests/unit/test_rerank_cache.py
 tests/unit/test_resolve.py
+tests/unit/test_retrieve_any_near_duplicates.py
 tests/unit/test_roadmap_fresh.py
 tests/unit/test_rules.py
 tests/unit/test_scale_corpus.py

{boost_skill_cli-1.2.72 → boost_skill_cli-1.2.74}/docs/commands.html

@@ -117,7 +117,7 @@
     </div>
     <div class="group" data-group="find">
       <div class="group-h">Find &amp; search</div>
-      <a class="cmd-link" href="#cmd-search" data-search="search search skills across tap registries (ai-ranked)  --smart --limit limit --json query">search</a>
+      <a class="cmd-link" href="#cmd-search" data-search="search search skills across tap registries (ai-ranked)  --smart --limit limit --collapse-near-duplicates --json query">search</a>
       <a class="cmd-link" href="#cmd-reindex" data-search="reindex build/refresh the full-content search index build or refresh the full-content search index --force --dense --export-shard tap --import-shard file --fetch-shards --json">reindex</a>
       <a class="cmd-link" href="#cmd-discover" data-search="discover search github for skill repos you have not tapped yet search github for skill repositories you have not tapped yet --limit limit --json --local query">discover</a>
       <a class="cmd-link" href="#cmd-recommend" data-search="recommend suggest skills based on your project&#x27;s tech stack  --path path --limit limit --json">recommend</a>
@@ -397,10 +397,10 @@
         </div>
       </section>
       <h2 class="group-h" id="grp-find">Find &amp; search</h2>
-      <section class="cmd" id="cmd-search" data-search="search search skills across tap registries (ai-ranked)  --smart --limit limit --json query">
+      <section class="cmd" id="cmd-search" data-search="search search skills across tap registries (ai-ranked)  --smart --limit limit --collapse-near-duplicates --json query">
         <h3><span class="cname">search</span><span class="gtag">find</span></h3>
         <p class="summary">Search skills across tap registries (AI-ranked)</p>
-        <pre class="syn">boost search [--smart] [--limit LIMIT] [--json] query [query ...]</pre>
+        <pre class="syn">boost search [--smart] [--limit LIMIT] [--collapse-near-duplicates] [--json] query [query ...]</pre>
         <div class="args-h">Arguments</div>
         <div class="args">
           <div class="arg"><code>query</code><span>search terms</span></div>
@@ -409,6 +409,7 @@
         <div class="args">
           <div class="arg"><code>--smart</code><span>rerank the top hits with Claude</span></div>
           <div class="arg"><code>--limit LIMIT</code><span>max results (default 15)</span></div>
+          <div class="arg"><code>--collapse-near-duplicates</code><span>collapse near-identical entries (translations, paraphrases) via dense embeddings — opt-in, needs a built dense index; threshold not yet validated against a real corpus, see the roadmap</span></div>
           <div class="arg"><code>--json</code><span>machine-readable output</span></div>
         </div>
       </section>

boost_skill_cli-1.2.74/docs/roadmap/items/near-identical-copies-still-eat-the-slots.md

@@ -0,0 +1,146 @@
+---
+id: near-identical-copies-still-eat-the-slots
+board: code
+section: planned
+status: inflight
+owner: loop/near-dup-bound
+category: Search · Ranking
+complexity: M
+impact: High
+wow: 4
+note: shipped opt-in (#639); bound measured (#645) and it refutes the zero-clusters test -- default-on now waits on a name-confusability veto
+order: 131
+title: Near-identical copies survive content-hash dedup and take the whole result page
+---
+<a href="#near-duplicate-items-eat-the-result-slots">Content-hash dedup</a> shipped and worked:
+<code>rag.dedupe_by_content</code> took duplicate result slots from <b>4.94 to 0.60 per query</b>
+over a 77-tap corpus. That card closed naming one thing still open &mdash; <em>near-identical
+rather than byte-identical clustering, where <code>core/typosquat.py</code>'s confusion machinery
+would apply</em> &mdash; and buried it under a <code>shipped</code> status where nobody would claim
+it. This card is that remainder, with a measurement that makes it look considerably worse than
+&ldquo;refinement&rdquo;.
+
+<b>Observed on a real 466-tap install</b> with hybrid RRF serving (658,131 chunks): for the query
+<code>exa search</code>, every one of the top ten rows is <code>exa-search</code>, and the
+descriptions are what give the
+shape away &mdash; one Japanese (<code>Exa MCPによるウェブ、コード、企業調査</code>), two Chinese
+(<code>通过Exa MCP进行神经搜索</code>), five English variants of <code>Neural search via Exa MCP</code>,
+plus <code>Use Exa MCP for current web&hellip;</code> and <code>AI-powered web search&hellip;</code>.
+All ten are <code>★ curated</code>. The footer reads
+<code>51 matches &middot; ranked by hybrid RRF (BM25 + dense)</code>.
+
+<b>Every one of those passed dedup correctly.</b> They are not byte-identical: they are the same
+skill in Japanese, in Chinese, and in five English phrasings across different registries. The body
+digest differs, so <code>dedupe_by_content</code> keeps them all &mdash; which is exactly the
+behaviour <code>#366</code> proved must be preserved, since two entries sharing a name can be
+genuinely different rules. The shipped fix is not misbehaving. It simply does not reach this shape.
+
+<b>What the 0.60 residual actually was.</b> The prior card described its leftover as &ldquo;entries
+sharing a <em>name</em> whose bodies genuinely differ, which must stay separate&rdquo; &mdash; true
+as stated, and it reads as a rounding error. At 466 taps the same residual is a full result page.
+The gap between 0.60 and 10.0 is worth understanding before designing anything: the 77-tap
+measurement used 50 natural-language queries averaged, and an <em>average</em> hides the shape here.
+Duplicate pressure was already known to be a step function of <em>which</em> registries are tapped
+rather than how many; near-identical pressure looks like a step function of <b>which query</b> &mdash;
+harmless across a query set, total on any query that lands on a widely-mirrored skill. Re-measure
+per-query maxima, not means.
+
+<b>The hard part is the safety proof, not the clustering.</b> Content hashing was adoptable because
+one count settled it: of 14,153 distinct bodies, clusters spanning more than one name numbered
+<b>zero</b>, so collapsing could not merge two different skills. Near-identical clustering has no
+such free proof &mdash; any similarity threshold loose enough to merge a Japanese translation with
+its English original is loose enough to merge two genuinely different skills that share boilerplate.
+Establish the equivalent bound first (over a real corpus, at the chosen threshold, count clusters
+spanning more than one <em>meaning</em>) or the fix trades a visible problem for a silent one.
+
+<b>Three things to get right.</b> <em>Translations are the motivating case and the hardest</em>: they
+share almost no tokens with the original, so token-overlap similarity will not find them while an
+embedding will &mdash; and the vectors are already on disk, which makes this cheaper here than it
+would be anywhere else. <em>Collapse before <code>k</code></em>, and at both the
+<code>retrieve</code> and <code>retrieve_any</code> seams, for the reason the shipped dedup already
+documents: fusion reintroduces copies either engine dropped, because the copies are distinct
+<code>(tap, skill_md)</code> keys and RRF has no reason to treat them as one. <em>The existing
+quality prior carries over unchanged</em> &mdash; <code>rag.source_rank</code> orders on the user's
+<code>curated</code> flag first and shipped <code>confidence</code> second, and choosing among
+near-identical copies is the same question as choosing among identical ones: where should the user
+install from.
+
+<b>Not to be confused with <code>#629</code></b>, which deduplicated vector <em>storage</em> (one
+row per distinct embedding, 39.7% repeats reclaimed). That is a disk-size fix beneath the index and
+changes no ranking; this is about which rows reach the user's screen.
+
+<b>What shipped, and what did not.</b> <code>rag.collapse_near_duplicate_hits</code> is the same
+"keep the earliest rank slot, promote a better source" contract as <code>dedupe_by_content</code>,
+run over cosine similarity of the entries' first-chunk embeddings
+(<code>dense.entry_vectors</code>, an index probe through <code>chunks_entry</code> on a quantized
+store) instead of a body hash, at the <code>retrieve_any</code> seam before <code>k</code> is
+applied. It is covered by unit tests down to the arithmetic (<code>_cosine</code>'s dimension-
+mismatch and zero-vector guards), the clustering contract (rank order, quality-prior promotion,
+limit-after-collapse), the <code>dense.entry_vectors</code> lookup against a real quantized
+sqlite-vec store, and the <code>retrieve_any</code>/<code>boost search
+--collapse-near-duplicates</code> wiring in both directions (on and off).
+
+It ships <b>opt-in and off by default</b> — <code>retrieve_any(..., collapse_near_duplicates=True)</code>
+or <code>boost search --collapse-near-duplicates</code> — rather than replacing
+<code>dedupe_by_content</code>'s output on the default path. Two things this card asks for are still
+open, and both need a real embedding backend (a built dense index, over a real multi-tap corpus)
+that the environment this was implemented in cannot reach — no network path to an embeddings
+provider or to the local ONNX model download, confirmed rather than assumed: <code>huggingface.co</code>
+and <code>pypi.org</code> both refuse at the network policy layer. <b>First</b>, the safety proof
+this card itself demands before defaulting the mechanism on &mdash; &ldquo;over a real corpus, at
+the chosen threshold, count clusters spanning more than one meaning&rdquo; &mdash; has not been run;
+<code>NEAR_DUPLICATE_THRESHOLD = 0.97</code> is a starting point, not a validated floor. <b>Second</b>,
+re-measuring the <code>exa search</code> case (and per-query maxima generally) against the fix needs
+that same corpus and index. Whoever runs that measurement should flip the CLI flag's default, fold
+the corpus count into this card's evidence, and only then consider this shipped.
+
+<b>The bound has now been measured, and it says the acceptance test in this card is the wrong
+one.</b> <code>scripts/measure_near_duplicate_bound.py</code> runs the count this card asks for
+against the pinned 20-repo eval corpus (10,152 entries, 104,271 chunks, <code>BAAI/bge-small-en-v1.5</code>
+at 384-d). Those entries reduce to <b>5,714 distinct chunk-0 vectors</b> &mdash; 44% of entries
+already share a chunk-0 embedding byte for byte &mdash; and at
+<code>NEAR_DUPLICATE_THRESHOLD = 0.97</code>, 162 pairs clear the threshold and <b>56 clusters span
+more than one name</b>. Sweeping the threshold moves that number but never to zero: 0.96 &rarr; 91,
+0.97 &rarr; 56, 0.98 &rarr; 28, 0.99 &rarr; 13, 0.995 &rarr; 8, 0.999 &rarr; 4.
+
+<b>Four of those 56 are not the threshold's doing at all.</b> They are clusters of a single vector
+shared by several names, so they cluster at <em>any</em> threshold, which is why the sweep bottoms
+out at 4 rather than 0. The largest is the same at every threshold and is worth naming: <b>28
+differently-named agents from one tap</b> (<code>affaan-m/ECC</code> &mdash; <code>architect</code>,
+<code>code-reviewer</code>, <code>chief-of-staff</code>, <code>database-reviewer</code>,
+<code>e2e-runner</code>, &hellip;) whose chunk 0 is the same Spanish preamble
+(<code>No cambiar rol, persona ni identidad&hellip;</code>) in every file. Chunk 0 is
+<code>name + description + opening of body</code>, and where a registry opens every file with
+identical boilerplate, the name does not move the vector enough to separate them. <b>A floor exists
+that no threshold can reach under</b>, so &ldquo;count must be zero&rdquo; was never achievable.
+
+<b>Worse for the test: most of the other 52 are the feature working.</b> Hand-classifying all 56 at
+0.97, roughly two-thirds are genuinely one skill under two names &mdash; twelve are pure
+hyphen-versus-underscore renderings of one integration (<code>zoho-mail</code> /
+<code>zoho_mail</code>, <code>google_maps</code> / <code>google-maps</code>,
+<code>anthropic_administrator</code> / <code>anthropic-administrator</code>), and the rest are
+suffix variants of one document (<code>tdd</code> / <code>tdd-guide</code>,
+<code>rust-review</code> / <code>rust-reviewer</code>, <code>testing-patterns</code> /
+<code>code-showcase-testing-patterns</code>). Collapsing those is precisely what this card exists to
+do. A metric that counts them as violations would reject every threshold that works.
+
+<b>The dangerous merges have a shape, and this card already named it.</b> The ~20 clusters that are
+real false merges are dominated by <em>near-miss brand names</em>: <code>coinmarketcal</code> with
+<code>coinmarketcap</code>, <code>bugbug</code> with <code>bugsnag</code>, <code>parsehub</code>
+with <code>parseur</code>, <code>linkhut</code> with <code>linkup</code>,
+<code>mx-technologies</code> with <code>mx-toolbox</code>,
+<code>salesforce-marketing-cloud</code> with <code>salesforce-service-cloud</code>. These are
+distinct products whose descriptions are boilerplate around a swapped word. That is the
+<code>core/typosquat.py</code> confusion shape this card's opening paragraph pointed at, arrived at
+independently from the other end: the guard this needs is not a tighter cosine floor but a
+<em>name-confusability veto</em> &mdash; refuse to collapse two entries whose names are a confusable
+edit apart, however close their vectors sit.
+
+<b>So the default stays off, for a better-supported reason than before.</b> The measurement does not
+say 0.97 is too loose; it says similarity alone cannot separate <code>tdd</code>/<code>tdd-guide</code>
+(collapse) from <code>coinmarketcal</code>/<code>coinmarketcap</code> (never collapse), because both
+pairs sit in the same cosine band. Flipping the default needs the confusability veto first, and a
+re-count with it applied. <b>And this bound is space-specific</b>: it was measured in
+<code>bge-small</code> 384-d, while a keyed production install is <code>voyage-4</code> at 1024-d.
+Cosine thresholds do not transfer between embedding spaces &mdash; rerun the script against each
+space before trusting a number in it.

{boost_skill_cli-1.2.72 → boost_skill_cli-1.2.74}/docs/roadmap.html

@@ -1063,7 +1063,7 @@ checks fails on every invocation is the more expensive half of this card.</p></d
       </article>
 
       <article class="cap rcard" id="near-identical-copies-still-eat-the-slots">
-        <div class="head"><span class="pill planned">Planned</span><span class="cat">Search · Ranking</span></div>
+        <div class="head"><span class="pill inflight">In flight</span><span class="cat">Search · Ranking</span></div>
         <h3>Near-identical copies survive content-hash dedup and take the whole result page</h3>
         <p><a href="#near-duplicate-items-eat-the-result-slots">Content-hash dedup</a> shipped and worked:
 <code>rag.dedupe_by_content</code> took duplicate result slots from <b>4.94 to 0.60 per query</b>
@@ -1120,12 +1120,88 @@ install from.
 
 <b>Not to be confused with <code>#629</code></b>, which deduplicated vector <em>storage</em> (one
 row per distinct embedding, 39.7% repeats reclaimed). That is a disk-size fix beneath the index and
-changes no ranking; this is about which rows reach the user's screen.</p>
+changes no ranking; this is about which rows reach the user's screen.
+
+<b>What shipped, and what did not.</b> <code>rag.collapse_near_duplicate_hits</code> is the same
+"keep the earliest rank slot, promote a better source" contract as <code>dedupe_by_content</code>,
+run over cosine similarity of the entries' first-chunk embeddings
+(<code>dense.entry_vectors</code>, an index probe through <code>chunks_entry</code> on a quantized
+store) instead of a body hash, at the <code>retrieve_any</code> seam before <code>k</code> is
+applied. It is covered by unit tests down to the arithmetic (<code>_cosine</code>'s dimension-
+mismatch and zero-vector guards), the clustering contract (rank order, quality-prior promotion,
+limit-after-collapse), the <code>dense.entry_vectors</code> lookup against a real quantized
+sqlite-vec store, and the <code>retrieve_any</code>/<code>boost search
+--collapse-near-duplicates</code> wiring in both directions (on and off).
+
+It ships <b>opt-in and off by default</b> — <code>retrieve_any(..., collapse_near_duplicates=True)</code>
+or <code>boost search --collapse-near-duplicates</code> — rather than replacing
+<code>dedupe_by_content</code>'s output on the default path. Two things this card asks for are still
+open, and both need a real embedding backend (a built dense index, over a real multi-tap corpus)
+that the environment this was implemented in cannot reach — no network path to an embeddings
+provider or to the local ONNX model download, confirmed rather than assumed: <code>huggingface.co</code>
+and <code>pypi.org</code> both refuse at the network policy layer. <b>First</b>, the safety proof
+this card itself demands before defaulting the mechanism on &mdash; &ldquo;over a real corpus, at
+the chosen threshold, count clusters spanning more than one meaning&rdquo; &mdash; has not been run;
+<code>NEAR_DUPLICATE_THRESHOLD = 0.97</code> is a starting point, not a validated floor. <b>Second</b>,
+re-measuring the <code>exa search</code> case (and per-query maxima generally) against the fix needs
+that same corpus and index. Whoever runs that measurement should flip the CLI flag's default, fold
+the corpus count into this card's evidence, and only then consider this shipped.
+
+<b>The bound has now been measured, and it says the acceptance test in this card is the wrong
+one.</b> <code>scripts/measure_near_duplicate_bound.py</code> runs the count this card asks for
+against the pinned 20-repo eval corpus (10,152 entries, 104,271 chunks, <code>BAAI/bge-small-en-v1.5</code>
+at 384-d). Those entries reduce to <b>5,714 distinct chunk-0 vectors</b> &mdash; 44% of entries
+already share a chunk-0 embedding byte for byte &mdash; and at
+<code>NEAR_DUPLICATE_THRESHOLD = 0.97</code>, 162 pairs clear the threshold and <b>56 clusters span
+more than one name</b>. Sweeping the threshold moves that number but never to zero: 0.96 &rarr; 91,
+0.97 &rarr; 56, 0.98 &rarr; 28, 0.99 &rarr; 13, 0.995 &rarr; 8, 0.999 &rarr; 4.
+
+<b>Four of those 56 are not the threshold's doing at all.</b> They are clusters of a single vector
+shared by several names, so they cluster at <em>any</em> threshold, which is why the sweep bottoms
+out at 4 rather than 0. The largest is the same at every threshold and is worth naming: <b>28
+differently-named agents from one tap</b> (<code>affaan-m/ECC</code> &mdash; <code>architect</code>,
+<code>code-reviewer</code>, <code>chief-of-staff</code>, <code>database-reviewer</code>,
+<code>e2e-runner</code>, &hellip;) whose chunk 0 is the same Spanish preamble
+(<code>No cambiar rol, persona ni identidad&hellip;</code>) in every file. Chunk 0 is
+<code>name + description + opening of body</code>, and where a registry opens every file with
+identical boilerplate, the name does not move the vector enough to separate them. <b>A floor exists
+that no threshold can reach under</b>, so &ldquo;count must be zero&rdquo; was never achievable.
+
+<b>Worse for the test: most of the other 52 are the feature working.</b> Hand-classifying all 56 at
+0.97, roughly two-thirds are genuinely one skill under two names &mdash; twelve are pure
+hyphen-versus-underscore renderings of one integration (<code>zoho-mail</code> /
+<code>zoho_mail</code>, <code>google_maps</code> / <code>google-maps</code>,
+<code>anthropic_administrator</code> / <code>anthropic-administrator</code>), and the rest are
+suffix variants of one document (<code>tdd</code> / <code>tdd-guide</code>,
+<code>rust-review</code> / <code>rust-reviewer</code>, <code>testing-patterns</code> /
+<code>code-showcase-testing-patterns</code>). Collapsing those is precisely what this card exists to
+do. A metric that counts them as violations would reject every threshold that works.
+
+<b>The dangerous merges have a shape, and this card already named it.</b> The ~20 clusters that are
+real false merges are dominated by <em>near-miss brand names</em>: <code>coinmarketcal</code> with
+<code>coinmarketcap</code>, <code>bugbug</code> with <code>bugsnag</code>, <code>parsehub</code>
+with <code>parseur</code>, <code>linkhut</code> with <code>linkup</code>,
+<code>mx-technologies</code> with <code>mx-toolbox</code>,
+<code>salesforce-marketing-cloud</code> with <code>salesforce-service-cloud</code>. These are
+distinct products whose descriptions are boilerplate around a swapped word. That is the
+<code>core/typosquat.py</code> confusion shape this card's opening paragraph pointed at, arrived at
+independently from the other end: the guard this needs is not a tighter cosine floor but a
+<em>name-confusability veto</em> &mdash; refuse to collapse two entries whose names are a confusable
+edit apart, however close their vectors sit.
+
+<b>So the default stays off, for a better-supported reason than before.</b> The measurement does not
+say 0.97 is too loose; it says similarity alone cannot separate <code>tdd</code>/<code>tdd-guide</code>
+(collapse) from <code>coinmarketcal</code>/<code>coinmarketcap</code> (never collapse), because both
+pairs sit in the same cosine band. Flipping the default needs the confusability veto first, and a
+re-count with it applied. <b>And this bound is space-specific</b>: it was measured in
+<code>bge-small</code> 384-d, while a keyed production install is <code>voyage-4</code> at 1024-d.
+Cosine thresholds do not transfer between embedding spaces &mdash; rerun the script against each
+space before trusting a number in it.</p>
         <div class="meta">
           <span class="m">Complexity <b>M</b></span>
           <span class="m hi">Impact <b>High</b></span>
           <span class="m">Wow <b class="wow">★★★★</b></span>
-          <span class="m">10 of 10 slots on a real 466-tap install, all past byte-identical dedup</span>
+          <span class="m">shipped opt-in (#639); bound measured (#645) and it refutes the zero-clusters test -- default-on now waits on a name-confusability veto</span>
         </div>
       </article>
       <!-- /ROADMAP:cards -->