boost-skill-cli 1.2.62__tar.gz → 1.2.63__tar.gz

{boost_skill_cli-1.2.62/boost_skill_cli.egg-info → boost_skill_cli-1.2.63}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: boost-skill-cli
-Version: 1.2.62
+Version: 1.2.63
 Summary: boost — Homebrew for AI coding skills
 Author: Jonathan Reyes
 License-Expression: Apache-2.0

{boost_skill_cli-1.2.62 → boost_skill_cli-1.2.63}/boost_cli/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '1.2.62'
-__version_tuple__ = version_tuple = (1, 2, 62)
+__version__ = version = '1.2.63'
+__version_tuple__ = version_tuple = (1, 2, 63)
 
 __commit_id__ = commit_id = None

{boost_skill_cli-1.2.62 → boost_skill_cli-1.2.63}/boost_cli/core/dense.py

@@ -15,6 +15,7 @@ are the expensive step, so an unchanged tap is never re-embedded.
 from __future__ import annotations
 
 import base64
+import hashlib
 import json
 import sqlite3
 from pathlib import Path
@@ -84,6 +85,27 @@ def _connect() -> sqlite3.Connection | None:
 # and returns the same 60 rows, so the extra candidates are pure overhead.
 RESCORE_POOL = 2048
 
+#: How many result entries may share one byte-identical embedding.
+#:
+#: Registries paste boilerplate. On a real 657,587-chunk store the largest
+#: cluster of byte-identical vectors is **1,464 copies spanning 1,464 distinct
+#: skills across 2 taps** — one Composio/Rube tool-calling paragraph, vendored
+#: into every skill in two registries. Identical vectors score an identical
+#: distance, so `retrieve`'s tie-break (the displayed name) decided the page:
+#: a query landing near that paragraph returned the alphabetically-first 60 of
+#: the 1,464, measured **60 of 60 slots**, and every one of them matched on
+#: text its skill did not write.
+#:
+#: `rag.dedupe_by_content` cannot reach this. It keys on the *entry* body
+#: digest, and these are 1,464 genuinely different entries — correctly kept
+#: apart. The repetition is one chunk *inside* each of them, which is a
+#: different axis and the one `near-duplicate-items-eat-the-result-slots`
+#: leaves open.
+#:
+#: Not 1, because two skills can legitimately share a paragraph that really is
+#: the best answer; a handful shows that without letting it own the page.
+MAX_PER_VECTOR = 3
+
 
 def _quantizable(dim: int) -> bool:
     """Whether ``dim`` can be binary-quantized at all.
@@ -1101,8 +1123,11 @@ def quantize() -> dict | None:
 
 
 def _knn(con: sqlite3.Connection, qblob: bytes,
-         pool: int) -> list[tuple[int, float]]:
-    """``(chunk_id, cosine_distance)`` for the ``pool`` nearest chunks.
+         pool: int) -> list[tuple[int, float, bytes]]:
+    """``(chunk_id, cosine_distance, vector_key)`` for the nearest chunks.
+
+    ``vector_key`` identifies the *embedding*, not the row: byte-identical
+    vectors share one key, which is what :data:`MAX_PER_VECTOR` caps on.
 
     Two-stage on a quantized store, one stage on a legacy one, and the answer
     is the same either way — which is the whole point. `vec0` has no ANN index,
@@ -1132,14 +1157,43 @@ def _knn(con: sqlite3.Connection, qblob: bytes,
         # Exact cosine over the candidates only. `vec_raw` is an ordinary
         # rowid-keyed table precisely so this `IN` is an index lookup; the same
         # clause against a vec0 table plans as a full scan (see _ensure_schema).
-        return con.execute(
-            "SELECT id, vec_distance_cosine(embedding, ?) AS d FROM vec_raw "  # noqa: S608  interpolates only `?` placeholders
-            "WHERE id IN (%s) ORDER BY d LIMIT ?"
-            % ",".join("?" * len(cand)), (qblob, *cand, pool)).fetchall()
-    return con.execute(
+        #
+        # The embedding comes back too, and it is not waste: hashing it is what
+        # lets `retrieve` tell "60 entries that each matched something" from
+        # "60 copies of one paragraph". The bytes were already read to compute
+        # the distance, so the only new cost is one sha256 per candidate.
+        # No LIMIT, and that is the point: sqlite computes a distance for all
+        # `cand` rows either way — a LIMIT only truncates what it hands back.
+        # Truncating here is what let one cluster own the pool. The largest on
+        # a real store is 1,464 copies of one pasted paragraph, so a 480-row
+        # cut was 480 copies of it and the page had nothing else to rank.
+        rows = con.execute(
+            "SELECT id, vec_distance_cosine(embedding, ?) AS d, embedding "  # noqa: S608  interpolates only `?` placeholders
+            "FROM vec_raw WHERE id IN (%s) ORDER BY d"
+            % ",".join("?" * len(cand)), (qblob, *cand)).fetchall()
+        # Thin to `pool` while letting no single embedding take more than
+        # MAX_PER_VECTOR of it. Rows arrive sorted, so the copies that survive
+        # are the nearest ones and the slots freed go to the next *distinct*
+        # vector rather than to the next copy of this one.
+        out: list[tuple[int, float, bytes]] = []
+        per: dict[bytes, int] = {}
+        for rid, dist, blob in rows:
+            if len(out) >= pool:
+                break
+            vkey = hashlib.sha256(blob).digest()[:16]
+            if per.get(vkey, 0) >= MAX_PER_VECTOR:
+                continue
+            per[vkey] = per.get(vkey, 0) + 1
+            out.append((rid, dist, vkey))
+        return out
+    # Legacy float32 layout: one row per chunk with no separate blob table, so
+    # there is nothing cheap to hash. Each row becomes its own group, which
+    # makes the cap a no-op — correct rather than clever, because this path
+    # exists for widths `bit[N]` cannot take and no real store is on it.
+    return [(rid, dist, b"%d" % rid) for rid, dist in con.execute(
         "SELECT rowid, distance FROM vec_chunks "
         "WHERE embedding MATCH ? ORDER BY distance LIMIT ?",
-        (qblob, pool)).fetchall()
+        (qblob, pool)).fetchall()]
 
 
 def retrieve(query: str, k: int = 60, kind: str | None = None,
@@ -1169,14 +1223,18 @@ def retrieve(query: str, k: int = 60, kind: str | None = None,
             return []
         by_id = {r[0]: (r[1], r[2], r[3], r[4]) for r in con.execute(
             "SELECT id, tap, path, kind, snip FROM chunks WHERE id IN (%s)"  # noqa: S608  interpolates only `?` placeholders; ids are bound params
-            % ",".join("?" * len(knn)), [rid for rid, _d in knn])}
+            % ",".join("?" * len(knn)), [rid for rid, _d, _v in knn])}
     finally:
         con.close()
 
     entries = catalog.all_entries() if entries is None else entries
     live = {entry_key(e): e for e in entries}
     best: dict[tuple[str, str], tuple[float, str]] = {}
-    for rid, dist in knn:
+    # Which embedding won each entry its score. An entry that also matched on
+    # a chunk of its own keeps *that* pairing, because `best` only records the
+    # winner — so capping below never costs an entry a hit it earned itself.
+    won_by: dict[tuple[str, str], bytes] = {}
+    for rid, dist, vkey in knn:
         meta = by_id.get(rid)
         if meta is None:
             continue
@@ -1190,11 +1248,26 @@ def retrieve(query: str, k: int = 60, kind: str | None = None,
         prev = best.get(key)
         if prev is None or score > prev[0]:
             best[key] = (score, snip)
+            won_by[key] = vkey
     # Tie-break on the displayed name (see rag.retrieve for why).
     ranked = sorted(best.items(),
                     key=lambda kv: (-kv[1][0], live[kv[0]]["name"], kv[0]))
-    hits: list[Hit] = [
-        {"entry": live[key], "score": score,
-         "snippet": snip}  # type: ignore[typeddict-item]
-        for key, (score, snip) in ranked[:k]]
+    hits: list[Hit] = []
+    seen_vec: dict[bytes, int] = {}
+    for key, (score, snip) in ranked:
+        if len(hits) >= k:
+            break
+        # The cap, and the reason the sort above is not enough on its own:
+        # byte-identical vectors produce a byte-identical distance, so every
+        # entry sharing one arrives with the *same* score and the tie-break
+        # decides the page alphabetically. Measured on a real store, one
+        # 1,464-copy cluster took 60 of 60 slots that way. Skipping past the
+        # cap rather than truncating keeps the page full: the freed slots go
+        # to the next entries that matched on something else.
+        vkey = won_by[key]
+        if seen_vec.get(vkey, 0) >= MAX_PER_VECTOR:
+            continue
+        seen_vec[vkey] = seen_vec.get(vkey, 0) + 1
+        hits.append({"entry": live[key], "score": score,
+                     "snippet": snip})  # type: ignore[typeddict-item]
     return hits

{boost_skill_cli-1.2.62 → boost_skill_cli-1.2.63/boost_skill_cli.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: boost-skill-cli
-Version: 1.2.62
+Version: 1.2.63
 Summary: boost — Homebrew for AI coding skills
 Author: Jonathan Reyes
 License-Expression: Apache-2.0

{boost_skill_cli-1.2.62 → boost_skill_cli-1.2.63}/boost_skill_cli.egg-info/SOURCES.txt

@@ -257,6 +257,7 @@ docs/roadmap/items/dense-reindex-has-no-progress-or-durability.md
 docs/roadmap/items/dense-reindex-reuses-whole-taps.md
 docs/roadmap/items/dense-search-fallback-and-stale-tap-pruning.md
 docs/roadmap/items/dense-search-rescans-every-vector.md
+docs/roadmap/items/dense-store-holds-every-vector-once-per-copy.md
 docs/roadmap/items/dependabot-missing-pip-ecosystem.md
 docs/roadmap/items/dependabot-regeneration-drops-platform-pins.md
 docs/roadmap/items/dependabot-root-pip-entry-duplicates-requirements.md
@@ -688,6 +689,7 @@ tests/unit/test_crash_redaction.py
 tests/unit/test_dedup_trust.py
 tests/unit/test_demo_index.py
 tests/unit/test_dense.py
+tests/unit/test_dense_boilerplate_cluster.py
 tests/unit/test_dense_embed_dedupe.py
 tests/unit/test_dense_entry_reuse.py
 tests/unit/test_dense_fallback.py

boost_skill_cli-1.2.63/docs/roadmap/items/dense-store-holds-every-vector-once-per-copy.md

@@ -0,0 +1,79 @@
+---
+id: dense-store-holds-every-vector-once-per-copy
+board: code
+section: internals
+status: planned
+category: Search · Storage
+complexity: L
+impact: Med
+wow: 3
+note: 39.7% of vector rows are byte-identical repeats; 1.43x whole-store, no re-embedding
+order: 128
+owner:
+pr:
+title: The dense store keeps one vector per <em>copy</em>, not one per distinct text
+---
+<code>_embed_and_store</code> already buys each distinct text <b>once</b> &mdash; <code>seen</code>
+de-duplicates before the provider call, which is why <code>build</code>'s progress total counts
+distinct texts rather than rows. The <em>storage</em> never got the same treatment: every chunk row
+still gets its own <code>vec_raw</code> blob and its own <code>vec_chunks_bin</code> row, so a text
+vendored into 1,464 skills is stored 1,464 times.
+
+<b>Measured on a real store</b> (657,587 chunks, 384-d <code>BAAI/bge-small-en-v1.5</code>):
+657,587 vector rows collapse to <b>396,638 distinct</b> &mdash; 39.7% repeats, <b>1.658&times;</b>.
+The earlier 750,416-chunk / 1024-d Voyage store measured 42.9%, so the ratio is a property of the
+corpus rather than of one embedder.
+
+<b>State the payoff in the right scope.</b> The <code>#544</code> note said &ldquo;~1.75&times;&rdquo;
+without saying of what, and that reads as a whole-store number. It is not. From <code>dbstat</code>
+on the live file: <code>vec_raw</code> 1,287.6&nbsp;MB and <code>vec_chunks_bin</code> 45.1&nbsp;MB
+of a 1,634&nbsp;MB total, so vectors are <b>81.6%</b> of the file. Deduplicating them saves
+<b>529&nbsp;MB</b>; add back a <code>vid</code> column, a hash index and a <code>chunks(vid)</code>
+index and the honest figure is <b>1,634 &rarr; ~1,140&nbsp;MB, 1.43&times; whole-store</b>. On a
+1024-d store, where vectors are ~90% of the file, it lands nearer 1.6&times;.
+
+<b>The key must be the embedding blob, not the text &mdash; and this is the constraint that decides
+the design.</b> <code>export_shard</code> emits <code>name/tap/path/kind/cix/snip/digest/embedding</code>
+and <code>snip</code> is <code>text[:200]</code>: <b>the full chunk text is not in the store and not
+in the shard</b>, so <code>import_shard</code> has nothing to hash. A <code>text_hash</code> column
+is unpopulatable on the import path. <code>sha256(embedding)</code> is equivalent by construction
+&mdash; identical texts are embedded once and serialize to identical bytes &mdash; and it works for
+imports and for the migration.
+
+<b>No join table.</b> <code>#544</code> proposed a chunk&rarr;tap join table; <code>chunks</code>
+already <em>is</em> that table and already carries <code>tap</code>. The change is one
+<code>vid</code> column on <code>chunks</code>, a <code>vectors</code> relation, and
+<code>vec_chunks_bin</code> keyed by <code>vid</code>.
+
+<b>Three things break, and one of them is an ordering.</b> <code>_delete_matching</code> drops
+vectors <em>then</em> rows; refcounted GC must run <em>after</em> the row delete or it counts
+references about to vanish. <code>_store_vector</code>'s orphan guard stops being sufficient, because
+under dedup a <code>_COMMIT_EVERY</code> interrupt makes orphans structural rather than exceptional.
+And <code>export_shard</code>'s <code>v.id = c.id</code> join becomes <code>v.vid = c.vid</code>
+&mdash; the same shape whose breakage the docstring already records as having killed two scheduled
+<code>shards</code> runs.
+
+<b>In place, not an <code>INDEX_VERSION</code> bump.</b> Every input is already on disk, so the
+migration re-encodes rather than re-embeds &mdash; the posture <code>quantize()</code> established,
+and roughly its cost (~1360&nbsp;s, ~2&times; peak while both copies coexist), except that this one
+ends <em>smaller</em>. <code>dense.py</code> promises in writing that v3's re-embed is
+&ldquo;the last one this mechanism needs&rdquo;; a v4 wipe would charge every user a full re-embed
+to reclaim disk, which is the wrong trade even where the embedder is free.
+
+<b>The quality half is already fixed, and this card is not blocked on it.</b>
+<code>MAX_PER_VECTOR</code> thins the candidate pool and caps the page, which is what stopped one
+1,464-copy cluster taking 60 of 60 result slots. Dedup would make the pool distinct
+<em>structurally</em> rather than by thinning, and would let <code>_knn</code> stop hashing 2,048
+blobs per query &mdash; a simplification, not a rescue.
+
+<b>The free bonus nobody has claimed:</b> <code>seen</code> in <code>_embed_and_store</code> is a
+local rebuilt per call, so reuse is <em>per build</em>. A persisted blob-keyed table makes it
+cross-build and cross-tap: a newly tapped mirror registry would cost zero embeddings.
+
+<b>Biggest risk, named:</b> a third schema state. The store already branches float32 vs quantized
+through every vector-touching function; &ldquo;deduplicated or not&rdquo; makes four combinations,
+each needing mutant-killing tests. And the one invariant that catches silent corruption &mdash;
+<code>test_dense_entry_reuse.py</code>'s two-way <code>chunks.id</code> &harr; <code>vec_raw.id</code>
+bijection &mdash; must be <b>rewritten</b> rather than extended, because dedup breaks one direction
+by design. Its replacement: every <code>chunks.vid</code> resolves, and every <code>vectors</code>
+row has at least one referent.

{boost_skill_cli-1.2.62 → boost_skill_cli-1.2.63}/docs/roadmap.html

@@ -133,7 +133,7 @@
       <div class="stat"><b>20</b><span>Shipped</span></div>
       <div class="stat"><b>1</b><span>Next up</span></div>
       <div class="stat"><b>7</b><span>Planned</span></div>
-      <div class="stat"><b>290</b><span>Loop finds</span></div>
+      <div class="stat"><b>291</b><span>Loop finds</span></div>
       <!-- /ROADMAP:stats -->
     </div>
   </section>
@@ -4886,6 +4886,81 @@ existing table, and <code>build</code> already wipes on a version change. One fu
           <span class="m">one changed file re-embeds 29 min for 1.9 s of new text</span>
         </div>
       </article>
+
+      <article class="cap rcard" id="dense-store-holds-every-vector-once-per-copy">
+        <div class="head"><span class="pill planned">Planned</span><span class="cat">Search · Storage</span></div>
+        <h3>The dense store keeps one vector per <em>copy</em>, not one per distinct text</h3>
+        <p><code>_embed_and_store</code> already buys each distinct text <b>once</b> &mdash; <code>seen</code>
+de-duplicates before the provider call, which is why <code>build</code>'s progress total counts
+distinct texts rather than rows. The <em>storage</em> never got the same treatment: every chunk row
+still gets its own <code>vec_raw</code> blob and its own <code>vec_chunks_bin</code> row, so a text
+vendored into 1,464 skills is stored 1,464 times.
+
+<b>Measured on a real store</b> (657,587 chunks, 384-d <code>BAAI/bge-small-en-v1.5</code>):
+657,587 vector rows collapse to <b>396,638 distinct</b> &mdash; 39.7% repeats, <b>1.658&times;</b>.
+The earlier 750,416-chunk / 1024-d Voyage store measured 42.9%, so the ratio is a property of the
+corpus rather than of one embedder.
+
+<b>State the payoff in the right scope.</b> The <code>#544</code> note said &ldquo;~1.75&times;&rdquo;
+without saying of what, and that reads as a whole-store number. It is not. From <code>dbstat</code>
+on the live file: <code>vec_raw</code> 1,287.6&nbsp;MB and <code>vec_chunks_bin</code> 45.1&nbsp;MB
+of a 1,634&nbsp;MB total, so vectors are <b>81.6%</b> of the file. Deduplicating them saves
+<b>529&nbsp;MB</b>; add back a <code>vid</code> column, a hash index and a <code>chunks(vid)</code>
+index and the honest figure is <b>1,634 &rarr; ~1,140&nbsp;MB, 1.43&times; whole-store</b>. On a
+1024-d store, where vectors are ~90% of the file, it lands nearer 1.6&times;.
+
+<b>The key must be the embedding blob, not the text &mdash; and this is the constraint that decides
+the design.</b> <code>export_shard</code> emits <code>name/tap/path/kind/cix/snip/digest/embedding</code>
+and <code>snip</code> is <code>text[:200]</code>: <b>the full chunk text is not in the store and not
+in the shard</b>, so <code>import_shard</code> has nothing to hash. A <code>text_hash</code> column
+is unpopulatable on the import path. <code>sha256(embedding)</code> is equivalent by construction
+&mdash; identical texts are embedded once and serialize to identical bytes &mdash; and it works for
+imports and for the migration.
+
+<b>No join table.</b> <code>#544</code> proposed a chunk&rarr;tap join table; <code>chunks</code>
+already <em>is</em> that table and already carries <code>tap</code>. The change is one
+<code>vid</code> column on <code>chunks</code>, a <code>vectors</code> relation, and
+<code>vec_chunks_bin</code> keyed by <code>vid</code>.
+
+<b>Three things break, and one of them is an ordering.</b> <code>_delete_matching</code> drops
+vectors <em>then</em> rows; refcounted GC must run <em>after</em> the row delete or it counts
+references about to vanish. <code>_store_vector</code>'s orphan guard stops being sufficient, because
+under dedup a <code>_COMMIT_EVERY</code> interrupt makes orphans structural rather than exceptional.
+And <code>export_shard</code>'s <code>v.id = c.id</code> join becomes <code>v.vid = c.vid</code>
+&mdash; the same shape whose breakage the docstring already records as having killed two scheduled
+<code>shards</code> runs.
+
+<b>In place, not an <code>INDEX_VERSION</code> bump.</b> Every input is already on disk, so the
+migration re-encodes rather than re-embeds &mdash; the posture <code>quantize()</code> established,
+and roughly its cost (~1360&nbsp;s, ~2&times; peak while both copies coexist), except that this one
+ends <em>smaller</em>. <code>dense.py</code> promises in writing that v3's re-embed is
+&ldquo;the last one this mechanism needs&rdquo;; a v4 wipe would charge every user a full re-embed
+to reclaim disk, which is the wrong trade even where the embedder is free.
+
+<b>The quality half is already fixed, and this card is not blocked on it.</b>
+<code>MAX_PER_VECTOR</code> thins the candidate pool and caps the page, which is what stopped one
+1,464-copy cluster taking 60 of 60 result slots. Dedup would make the pool distinct
+<em>structurally</em> rather than by thinning, and would let <code>_knn</code> stop hashing 2,048
+blobs per query &mdash; a simplification, not a rescue.
+
+<b>The free bonus nobody has claimed:</b> <code>seen</code> in <code>_embed_and_store</code> is a
+local rebuilt per call, so reuse is <em>per build</em>. A persisted blob-keyed table makes it
+cross-build and cross-tap: a newly tapped mirror registry would cost zero embeddings.
+
+<b>Biggest risk, named:</b> a third schema state. The store already branches float32 vs quantized
+through every vector-touching function; &ldquo;deduplicated or not&rdquo; makes four combinations,
+each needing mutant-killing tests. And the one invariant that catches silent corruption &mdash;
+<code>test_dense_entry_reuse.py</code>'s two-way <code>chunks.id</code> &harr; <code>vec_raw.id</code>
+bijection &mdash; must be <b>rewritten</b> rather than extended, because dedup breaks one direction
+by design. Its replacement: every <code>chunks.vid</code> resolves, and every <code>vectors</code>
+row has at least one referent.</p>
+        <div class="meta">
+          <span class="m">Complexity <b>L</b></span>
+          <span class="m">Impact <b>Med</b></span>
+          <span class="m">Wow <b class="wow">★★★</b></span>
+          <span class="m">39.7% of vector rows are byte-identical repeats; 1.43x whole-store, no re-embedding</span>
+        </div>
+      </article>
       <!-- /ROADMAP:cards -->
 
     </div>

boost_skill_cli-1.2.63/tests/unit/test_dense_boilerplate_cluster.py

@@ -0,0 +1,229 @@
+# Copyright the boost contributors.
+# SPDX-License-Identifier: Apache-2.0
+"""One pasted paragraph must not own the result page.
+
+Registries vendor boilerplate. Measured on a real 657,587-chunk store, the
+largest cluster of **byte-identical** embeddings is 1,464 copies spanning
+**1,464 distinct skills across two taps** — a Composio/Rube tool-calling
+paragraph pasted into every skill in two registries.
+
+Identical vectors produce an identical cosine distance, so every entry holding
+that paragraph arrived with the *same* score and `retrieve`'s tie-break (the
+displayed name) decided the page alphabetically. Measured before this change:
+a query landing near that paragraph filled **60 of 60 slots** from the cluster.
+Not one of those results matched on text its own skill wrote.
+
+`rag.dedupe_by_content` cannot reach it, and is right not to: it keys on the
+*entry body* digest, and these are 1,464 genuinely different entries. The
+repetition is one chunk *inside* each of them — the axis
+`near-duplicate-items-eat-the-result-slots` leaves open.
+
+**Capping the page alone is not enough, and that is the subtle part.** The
+candidate pool is built first: the rescore used to take the 480 nearest rows,
+and 480 of the cluster's 1,464 copies are nearer than anything else, so the
+pool was 100% one vector. Capping the page then yielded three results and
+57 empty slots. Measured on the real store, before and after:
+
+    pool distinct vectors     1  ->  242
+    rows from the cluster   480  ->    3
+    final page       60/60 boilerplate  ->  60 results, 30 distinct vectors
+
+So the cap has to run **while the pool is being assembled**, where the rows
+that lose a slot can be replaced by the next distinct vector rather than by
+the next copy. `_knn` thins; `retrieve` caps entries. Different units, both
+needed: one chunk can serve many entries, and one entry can hold many chunks.
+"""
+import math
+import sqlite3
+
+import pytest
+
+from boost_cli.core import dense, embed, rag
+
+
+def _vec_loadable() -> bool:
+    con = sqlite3.connect(":memory:")
+    try:
+        import sqlite_vec
+        con.enable_load_extension(True)
+        sqlite_vec.load(con)
+        con.enable_load_extension(False)
+        con.execute("create virtual table t using vec0(embedding bit[8])")
+        return True
+    except Exception:
+        return False
+    finally:
+        con.close()
+
+
+pytestmark = pytest.mark.skipif(
+    not _vec_loadable(), reason="sqlite-vec extension not loadable here")
+
+#: Twelve skills paste the SAME paragraph; four have bodies of their own. The
+#: shape of the real finding at a size a unit test can hold.
+_SHARED = "shared-boilerplate-paragraph"
+_PASTERS = ["paster-%02d" % i for i in range(12)]
+_ORIGINALS = ["original-%02d" % i for i in range(4)]
+
+
+def _unit(v):
+    n = math.sqrt(sum(x * x for x in v)) or 1.0
+    return [x / n for x in v]
+
+
+def _vec_for(seed: int):
+    return _unit([math.sin(seed * 1.7 + i * 0.9) for i in range(8)])
+
+
+_BOILERPLATE_VEC = _vec_for(0)
+_ORIGINAL_VEC = {n: _vec_for(i + 1) for i, n in enumerate(_ORIGINALS)}
+
+
+def _toy_embed(texts, input_type=None, timeout=60):
+    out = []
+    for t in texts:
+        head = t.split("\n", 1)[0].strip()
+        # Every paster embeds to the IDENTICAL vector — that is the premise.
+        out.append(_ORIGINAL_VEC.get(head, _BOILERPLATE_VEC))
+    return out
+
+
+def _e(name):
+    return {"name": name, "tap": "acme/skills", "kind": "skill",
+            "skill_md": "skills/%s/SKILL.md" % name}
+
+
+_ENTRIES = [_e(n) for n in _PASTERS + _ORIGINALS]
+
+
+@pytest.fixture()
+def cluster_env(sandbox, monkeypatch):
+    monkeypatch.setattr(embed, "embed", _toy_embed)
+    monkeypatch.setattr(embed, "provider", lambda: "openai")
+    monkeypatch.setattr(embed, "model", lambda: "toy-8")
+    monkeypatch.setattr(embed, "dimension", lambda: 8)
+    monkeypatch.setattr(embed, "available", lambda: True)
+    monkeypatch.setattr(rag, "_tap_paths", lambda: {"acme/skills": "/x"})
+    monkeypatch.setattr(rag, "_tap_commits", lambda: {"acme__skills": "c1"})
+    # A paster's body IS the boilerplate; an original's body is its own name.
+    monkeypatch.setattr(
+        dense, "read_body",
+        lambda e, tp=None: (e["name"] if e["name"] in _ORIGINAL_VEC
+                            else _SHARED))
+    dense.build(entries=_ENTRIES, force=True)
+    return monkeypatch
+
+
+def _open():
+    con = sqlite3.connect(str(dense.db_path()))
+    import sqlite_vec
+    con.enable_load_extension(True)
+    sqlite_vec.load(con)
+    con.enable_load_extension(False)
+    return con
+
+
+def _qblob(con, vec):
+    mod = dense._load()
+    return mod.serialize_float32(vec)
+
+
+class TestThePremiseHolds:
+    """Without this the rest could pass by the cluster not existing."""
+
+    def test_the_pasters_really_do_share_one_embedding(self, cluster_env):
+        con = _open()
+        try:
+            blobs = {r[0] for r in con.execute("SELECT embedding FROM vec_raw")}
+            rows = con.execute("SELECT COUNT(*) FROM vec_raw").fetchone()[0]
+        finally:
+            con.close()
+        assert rows == len(_ENTRIES)
+        # 12 pasters collapse to one distinct blob; 4 originals keep their own.
+        assert len(blobs) == 1 + len(_ORIGINALS), len(blobs)
+
+
+class TestThePoolIsThinnedByVector:
+    def test_one_vector_cannot_take_the_whole_pool(self, cluster_env):
+        con = _open()
+        try:
+            knn = dense._knn(con, _qblob(con, _BOILERPLATE_VEC), pool=16)
+        finally:
+            con.close()
+        from collections import Counter
+        per = Counter(v for _i, _d, v in knn)
+        assert max(per.values()) <= dense.MAX_PER_VECTOR, per
+        # and the freed slots went to real, different vectors
+        assert len(per) > 1, per
+
+    def test_the_originals_are_still_reachable(self, cluster_env):
+        # The whole point of freeing the slots: entries that matched on
+        # something of their own must survive the cluster's neighbours.
+        con = _open()
+        try:
+            knn = dense._knn(con, _qblob(con, _BOILERPLATE_VEC), pool=16)
+            ids = [r[0] for r in knn]
+            names = {r[0] for r in con.execute(
+                "SELECT name FROM chunks WHERE id IN (%s)"  # noqa: S608  ids are bound params
+                % ",".join("?" * len(ids)), ids)}
+        finally:
+            con.close()
+        assert names & set(_ORIGINALS), names
+
+    def test_every_row_carries_its_vector_key(self, cluster_env):
+        con = _open()
+        try:
+            knn = dense._knn(con, _qblob(con, _BOILERPLATE_VEC), pool=8)
+        finally:
+            con.close()
+        assert knn and all(isinstance(v, bytes) and v for _i, _d, v in knn)
+
+
+class TestThePageIsCapped:
+    def test_the_cluster_does_not_own_the_results(self, cluster_env):
+        hits = dense.retrieve(_SHARED, k=12, entries=_ENTRIES)
+        assert hits is not None
+        pasters = [h for h in hits if h["entry"]["name"] in _PASTERS]
+        assert len(pasters) <= dense.MAX_PER_VECTOR, [
+            h["entry"]["name"] for h in hits]
+
+    def test_the_page_is_not_left_short_by_the_cap(self, cluster_env):
+        # Skipping past the cap rather than truncating is what keeps the page
+        # full — the bug the first draft of this fix had.
+        hits = dense.retrieve(_SHARED, k=6, entries=_ENTRIES)
+        assert hits is not None and len(hits) == 6, hits
+
+    def test_an_entry_that_matched_on_its_own_body_is_not_capped_away(
+            self, cluster_env):
+        name = _ORIGINALS[0]
+        hits = dense.retrieve(name, k=6, entries=_ENTRIES)
+        assert hits is not None
+        assert hits[0]["entry"]["name"] == name, [
+            h["entry"]["name"] for h in hits]
+
+    def test_a_query_with_no_cluster_is_unaffected(self, cluster_env):
+        # The cap must be invisible when nothing repeats: four originals, four
+        # distinct vectors, all four returned.
+        hits = dense.retrieve(_ORIGINALS[1], k=16, entries=_ENTRIES)
+        assert hits is not None
+        got = {h["entry"]["name"] for h in hits}
+        assert set(_ORIGINALS) <= got, got
+
+
+class TestTheCapIsAConstantNotAMagicNumber:
+    def test_it_is_declared_and_positive(self):
+        assert isinstance(dense.MAX_PER_VECTOR, int)
+        assert dense.MAX_PER_VECTOR >= 1
+
+    def test_raising_it_admits_more_copies(self, cluster_env, monkeypatch):
+        # Pins that the constant is what does the work — a mutant that ignores
+        # it, or hardcodes 3 elsewhere, fails here.
+        monkeypatch.setattr(dense, "MAX_PER_VECTOR", 8)
+        con = _open()
+        try:
+            knn = dense._knn(con, _qblob(con, _BOILERPLATE_VEC), pool=16)
+        finally:
+            con.close()
+        from collections import Counter
+        per = Counter(v for _i, _d, v in knn)
+        assert max(per.values()) > 3, per