boost-skill-cli 1.2.61__tar.gz → 1.2.63__tar.gz

{boost_skill_cli-1.2.61/boost_skill_cli.egg-info → boost_skill_cli-1.2.63}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: boost-skill-cli
-Version: 1.2.61
+Version: 1.2.63
 Summary: boost — Homebrew for AI coding skills
 Author: Jonathan Reyes
 License-Expression: Apache-2.0

{boost_skill_cli-1.2.61 → boost_skill_cli-1.2.63}/boost_cli/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '1.2.61'
-__version_tuple__ = version_tuple = (1, 2, 61)
+__version__ = version = '1.2.63'
+__version_tuple__ = version_tuple = (1, 2, 63)
 
 __commit_id__ = commit_id = None

{boost_skill_cli-1.2.61 → boost_skill_cli-1.2.63}/boost_cli/core/dense.py

@@ -15,6 +15,7 @@ are the expensive step, so an unchanged tap is never re-embedded.
 from __future__ import annotations
 
 import base64
+import hashlib
 import json
 import sqlite3
 from pathlib import Path
@@ -84,6 +85,27 @@ def _connect() -> sqlite3.Connection | None:
 # and returns the same 60 rows, so the extra candidates are pure overhead.
 RESCORE_POOL = 2048
 
+#: How many result entries may share one byte-identical embedding.
+#:
+#: Registries paste boilerplate. On a real 657,587-chunk store the largest
+#: cluster of byte-identical vectors is **1,464 copies spanning 1,464 distinct
+#: skills across 2 taps** — one Composio/Rube tool-calling paragraph, vendored
+#: into every skill in two registries. Identical vectors score an identical
+#: distance, so `retrieve`'s tie-break (the displayed name) decided the page:
+#: a query landing near that paragraph returned the alphabetically-first 60 of
+#: the 1,464, measured **60 of 60 slots**, and every one of them matched on
+#: text its skill did not write.
+#:
+#: `rag.dedupe_by_content` cannot reach this. It keys on the *entry* body
+#: digest, and these are 1,464 genuinely different entries — correctly kept
+#: apart. The repetition is one chunk *inside* each of them, which is a
+#: different axis and the one `near-duplicate-items-eat-the-result-slots`
+#: leaves open.
+#:
+#: Not 1, because two skills can legitimately share a paragraph that really is
+#: the best answer; a handful shows that without letting it own the page.
+MAX_PER_VECTOR = 3
+
 
 def _quantizable(dim: int) -> bool:
     """Whether ``dim`` can be binary-quantized at all.
@@ -302,7 +324,17 @@ def tap_commits() -> dict[str, str]:
     Read without sqlite-vec (see :func:`_recorded_meta`) so a machine that lost
     the extra still gets a truthful answer rather than an empty one.
     """
-    commits = _recorded_meta().get("commits")
+    meta = _recorded_meta()
+    if meta.get("version") != INDEX_VERSION:
+        # A store from an older boost is about to be discarded — `build` wipes
+        # it and so does `import_shard` — so it holds no vectors anyone can
+        # reuse, and saying otherwise is worse than saying nothing. The caller
+        # asking this question is deciding which taps to SKIP: answering with a
+        # stale store's commits let `ingest` skip 417 taps as "already current"
+        # and then wipe their vectors on the first import, leaving a store that
+        # was silently missing them while the output said they were fine.
+        return {}
+    commits = meta.get("commits")
     if not isinstance(commits, dict):
         return {}
     return {str(k): str(v) for k, v in commits.items() if v}
@@ -766,6 +798,18 @@ def import_shard(shard: dict, commit: str) -> tuple[bool, str]:
         return False, "no vector backend available"
     try:
         meta = _read_meta(con)
+        if meta.get("version") not in (None, INDEX_VERSION):
+            # The same wipe `build` does, for the same reason: `_ensure_schema`
+            # is CREATE TABLE IF NOT EXISTS, so it cannot add v3's `digest`
+            # column to a table built by an older boost, and the INSERT below
+            # would fail per row with "table chunks has no column named
+            # digest". On a real machine that is 449 shards reporting a sqlite
+            # message about a column, where the honest answer is that the store
+            # predates this format. Wiping loses nothing that was in use — a
+            # stale-version store is already dead weight, refused by `ready()`
+            # and wiped by `build()` on its next run.
+            _wipe(con)
+            meta = {}
         # An empty store has no opinion yet, so it adopts the shard's backend.
         if meta.get("provider"):
             for field in ("provider", "model", "dim"):
@@ -1079,8 +1123,11 @@ def quantize() -> dict | None:
 
 
 def _knn(con: sqlite3.Connection, qblob: bytes,
-         pool: int) -> list[tuple[int, float]]:
-    """``(chunk_id, cosine_distance)`` for the ``pool`` nearest chunks.
+         pool: int) -> list[tuple[int, float, bytes]]:
+    """``(chunk_id, cosine_distance, vector_key)`` for the nearest chunks.
+
+    ``vector_key`` identifies the *embedding*, not the row: byte-identical
+    vectors share one key, which is what :data:`MAX_PER_VECTOR` caps on.
 
     Two-stage on a quantized store, one stage on a legacy one, and the answer
     is the same either way — which is the whole point. `vec0` has no ANN index,
@@ -1110,14 +1157,43 @@ def _knn(con: sqlite3.Connection, qblob: bytes,
         # Exact cosine over the candidates only. `vec_raw` is an ordinary
         # rowid-keyed table precisely so this `IN` is an index lookup; the same
         # clause against a vec0 table plans as a full scan (see _ensure_schema).
-        return con.execute(
-            "SELECT id, vec_distance_cosine(embedding, ?) AS d FROM vec_raw "  # noqa: S608  interpolates only `?` placeholders
-            "WHERE id IN (%s) ORDER BY d LIMIT ?"
-            % ",".join("?" * len(cand)), (qblob, *cand, pool)).fetchall()
-    return con.execute(
+        #
+        # The embedding comes back too, and it is not waste: hashing it is what
+        # lets `retrieve` tell "60 entries that each matched something" from
+        # "60 copies of one paragraph". The bytes were already read to compute
+        # the distance, so the only new cost is one sha256 per candidate.
+        # No LIMIT, and that is the point: sqlite computes a distance for all
+        # `cand` rows either way — a LIMIT only truncates what it hands back.
+        # Truncating here is what let one cluster own the pool. The largest on
+        # a real store is 1,464 copies of one pasted paragraph, so a 480-row
+        # cut was 480 copies of it and the page had nothing else to rank.
+        rows = con.execute(
+            "SELECT id, vec_distance_cosine(embedding, ?) AS d, embedding "  # noqa: S608  interpolates only `?` placeholders
+            "FROM vec_raw WHERE id IN (%s) ORDER BY d"
+            % ",".join("?" * len(cand)), (qblob, *cand)).fetchall()
+        # Thin to `pool` while letting no single embedding take more than
+        # MAX_PER_VECTOR of it. Rows arrive sorted, so the copies that survive
+        # are the nearest ones and the slots freed go to the next *distinct*
+        # vector rather than to the next copy of this one.
+        out: list[tuple[int, float, bytes]] = []
+        per: dict[bytes, int] = {}
+        for rid, dist, blob in rows:
+            if len(out) >= pool:
+                break
+            vkey = hashlib.sha256(blob).digest()[:16]
+            if per.get(vkey, 0) >= MAX_PER_VECTOR:
+                continue
+            per[vkey] = per.get(vkey, 0) + 1
+            out.append((rid, dist, vkey))
+        return out
+    # Legacy float32 layout: one row per chunk with no separate blob table, so
+    # there is nothing cheap to hash. Each row becomes its own group, which
+    # makes the cap a no-op — correct rather than clever, because this path
+    # exists for widths `bit[N]` cannot take and no real store is on it.
+    return [(rid, dist, b"%d" % rid) for rid, dist in con.execute(
         "SELECT rowid, distance FROM vec_chunks "
         "WHERE embedding MATCH ? ORDER BY distance LIMIT ?",
-        (qblob, pool)).fetchall()
+        (qblob, pool)).fetchall()]
 
 
 def retrieve(query: str, k: int = 60, kind: str | None = None,
@@ -1147,14 +1223,18 @@ def retrieve(query: str, k: int = 60, kind: str | None = None,
             return []
         by_id = {r[0]: (r[1], r[2], r[3], r[4]) for r in con.execute(
             "SELECT id, tap, path, kind, snip FROM chunks WHERE id IN (%s)"  # noqa: S608  interpolates only `?` placeholders; ids are bound params
-            % ",".join("?" * len(knn)), [rid for rid, _d in knn])}
+            % ",".join("?" * len(knn)), [rid for rid, _d, _v in knn])}
     finally:
         con.close()
 
     entries = catalog.all_entries() if entries is None else entries
     live = {entry_key(e): e for e in entries}
     best: dict[tuple[str, str], tuple[float, str]] = {}
-    for rid, dist in knn:
+    # Which embedding won each entry its score. An entry that also matched on
+    # a chunk of its own keeps *that* pairing, because `best` only records the
+    # winner — so capping below never costs an entry a hit it earned itself.
+    won_by: dict[tuple[str, str], bytes] = {}
+    for rid, dist, vkey in knn:
         meta = by_id.get(rid)
         if meta is None:
             continue
@@ -1168,11 +1248,26 @@ def retrieve(query: str, k: int = 60, kind: str | None = None,
         prev = best.get(key)
         if prev is None or score > prev[0]:
             best[key] = (score, snip)
+            won_by[key] = vkey
     # Tie-break on the displayed name (see rag.retrieve for why).
     ranked = sorted(best.items(),
                     key=lambda kv: (-kv[1][0], live[kv[0]]["name"], kv[0]))
-    hits: list[Hit] = [
-        {"entry": live[key], "score": score,
-         "snippet": snip}  # type: ignore[typeddict-item]
-        for key, (score, snip) in ranked[:k]]
+    hits: list[Hit] = []
+    seen_vec: dict[bytes, int] = {}
+    for key, (score, snip) in ranked:
+        if len(hits) >= k:
+            break
+        # The cap, and the reason the sort above is not enough on its own:
+        # byte-identical vectors produce a byte-identical distance, so every
+        # entry sharing one arrives with the *same* score and the tie-break
+        # decides the page alphabetically. Measured on a real store, one
+        # 1,464-copy cluster took 60 of 60 slots that way. Skipping past the
+        # cap rather than truncating keeps the page full: the freed slots go
+        # to the next entries that matched on something else.
+        vkey = won_by[key]
+        if seen_vec.get(vkey, 0) >= MAX_PER_VECTOR:
+            continue
+        seen_vec[vkey] = seen_vec.get(vkey, 0) + 1
+        hits.append({"entry": live[key], "score": score,
+                     "snippet": snip})  # type: ignore[typeddict-item]
     return hits

{boost_skill_cli-1.2.61 → boost_skill_cli-1.2.63}/boost_cli/core/shards.py

@@ -94,6 +94,67 @@ def _open(url: str, timeout: float):
                               "embeds locally instead") from exc
 
 
+def _read_capped(resp, cap: int) -> bytes:
+    """Read the whole response, stopping once it exceeds `cap` bytes.
+
+    THE LOOP IS THE POINT. ``HTTPResponse.read(amt)`` returns *up to* `amt`
+    bytes, not `amt` bytes, and over a real connection it routinely returns
+    less — one read of the live 166 KB manifest came back with 131,072 bytes
+    on one run and 146,547 on the next. A single ``resp.read(cap + 1)``
+    therefore handed a truncated document to ``json.loads``, which reported it
+    as "not valid JSON": a network artefact wearing the costume of a corrupt
+    publish, on every command that reads the manifest.
+
+    The unit suite could not see it, and still cannot without help: its
+    fixtures are ``file:`` URLs, where one read does return the whole file.
+    ``tests/unit/test_shards.py`` covers this with a response that short-reads
+    on purpose.
+
+    Reading `cap + 1` in total keeps the size check that follows honest — one
+    byte over the ceiling is enough to fail it, and nothing larger is buffered.
+    """
+    chunks: list[bytes] = []
+    total = 0
+    while total <= cap:
+        buf = resp.read(min(_CHUNK, cap + 1 - total))
+        if not buf:
+            break
+        chunks.append(buf)
+        total += len(buf)
+    return b"".join(chunks)
+
+
+def _verify_complete(resp, raw: bytes, url: str) -> None:
+    """Refuse a body that arrived shorter than the server said it would.
+
+    ``read(amt)`` performs no length check — that is done only by ``read()``
+    with no argument — so a connection cut mid-stream returns a short body and
+    then a clean EOF. The truncated document reaches ``json.loads``, which
+    calls it "not valid JSON", and the user goes looking for a corrupt publish
+    that does not exist. Observed here against the live 166,210-byte manifest:
+    a read returned 146,547 bytes and the next returned nothing.
+
+    The Content-Encoding guard is not defensive padding. ``Content-Length``
+    describes the bytes ON THE WIRE, so against a gzipping proxy it is the
+    compressed size while `raw` is the decoded body — comparing the two would
+    manufacture a truncation failure, which is the same class of bug this
+    function exists to remove. A missing or unparseable header means the
+    question cannot be answered and is left alone.
+    """
+    enc = (resp.headers.get("Content-Encoding") or "").strip().lower()
+    if enc and enc != "identity":
+        return
+    try:
+        want = int(resp.headers.get("Content-Length"))
+    except (TypeError, ValueError):
+        return
+    if len(raw) < want:
+        raise BoostError(
+            "shard manifest download from %s was truncated (%d of %d bytes)"
+            % (url, len(raw), want),
+            hint="a proxy or a dropped connection cut the stream — retry")
+
+
 def fetch_manifest(url: str | None = None, timeout: float = 30.0) -> dict:
     """Download and validate the shard manifest.
 
@@ -104,9 +165,13 @@ def fetch_manifest(url: str | None = None, timeout: float = 30.0) -> dict:
     """
     url = url or manifest_url()
     with _open(url, timeout) as resp:
-        raw = resp.read(MAX_MANIFEST_BYTES + 1)
-    if len(raw) > MAX_MANIFEST_BYTES:
-        raise BoostError("shard manifest at %s is implausibly large" % url)
+        raw = _read_capped(resp, MAX_MANIFEST_BYTES)
+        # Size ceiling first, completeness second: a manifest larger than the
+        # cap is also "shorter than declared", and reporting that one as a
+        # truncated download would name the wrong problem.
+        if len(raw) > MAX_MANIFEST_BYTES:
+            raise BoostError("shard manifest at %s is implausibly large" % url)
+        _verify_complete(resp, raw, url)
     try:
         data = json.loads(raw.decode("utf-8"))
     except (UnicodeDecodeError, json.JSONDecodeError) as exc:

{boost_skill_cli-1.2.61 → boost_skill_cli-1.2.63/boost_skill_cli.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: boost-skill-cli
-Version: 1.2.61
+Version: 1.2.63
 Summary: boost — Homebrew for AI coding skills
 Author: Jonathan Reyes
 License-Expression: Apache-2.0

{boost_skill_cli-1.2.61 → boost_skill_cli-1.2.63}/boost_skill_cli.egg-info/SOURCES.txt

@@ -257,6 +257,7 @@ docs/roadmap/items/dense-reindex-has-no-progress-or-durability.md
 docs/roadmap/items/dense-reindex-reuses-whole-taps.md
 docs/roadmap/items/dense-search-fallback-and-stale-tap-pruning.md
 docs/roadmap/items/dense-search-rescans-every-vector.md
+docs/roadmap/items/dense-store-holds-every-vector-once-per-copy.md
 docs/roadmap/items/dependabot-missing-pip-ecosystem.md
 docs/roadmap/items/dependabot-regeneration-drops-platform-pins.md
 docs/roadmap/items/dependabot-root-pip-entry-duplicates-requirements.md
@@ -688,6 +689,7 @@ tests/unit/test_crash_redaction.py
 tests/unit/test_dedup_trust.py
 tests/unit/test_demo_index.py
 tests/unit/test_dense.py
+tests/unit/test_dense_boilerplate_cluster.py
 tests/unit/test_dense_embed_dedupe.py
 tests/unit/test_dense_entry_reuse.py
 tests/unit/test_dense_fallback.py

boost_skill_cli-1.2.63/docs/roadmap/items/dense-store-holds-every-vector-once-per-copy.md

@@ -0,0 +1,79 @@
+---
+id: dense-store-holds-every-vector-once-per-copy
+board: code
+section: internals
+status: planned
+category: Search · Storage
+complexity: L
+impact: Med
+wow: 3
+note: 39.7% of vector rows are byte-identical repeats; 1.43x whole-store, no re-embedding
+order: 128
+owner:
+pr:
+title: The dense store keeps one vector per <em>copy</em>, not one per distinct text
+---
+<code>_embed_and_store</code> already buys each distinct text <b>once</b> &mdash; <code>seen</code>
+de-duplicates before the provider call, which is why <code>build</code>'s progress total counts
+distinct texts rather than rows. The <em>storage</em> never got the same treatment: every chunk row
+still gets its own <code>vec_raw</code> blob and its own <code>vec_chunks_bin</code> row, so a text
+vendored into 1,464 skills is stored 1,464 times.
+
+<b>Measured on a real store</b> (657,587 chunks, 384-d <code>BAAI/bge-small-en-v1.5</code>):
+657,587 vector rows collapse to <b>396,638 distinct</b> &mdash; 39.7% repeats, <b>1.658&times;</b>.
+The earlier 750,416-chunk / 1024-d Voyage store measured 42.9%, so the ratio is a property of the
+corpus rather than of one embedder.
+
+<b>State the payoff in the right scope.</b> The <code>#544</code> note said &ldquo;~1.75&times;&rdquo;
+without saying of what, and that reads as a whole-store number. It is not. From <code>dbstat</code>
+on the live file: <code>vec_raw</code> 1,287.6&nbsp;MB and <code>vec_chunks_bin</code> 45.1&nbsp;MB
+of a 1,634&nbsp;MB total, so vectors are <b>81.6%</b> of the file. Deduplicating them saves
+<b>529&nbsp;MB</b>; add back a <code>vid</code> column, a hash index and a <code>chunks(vid)</code>
+index and the honest figure is <b>1,634 &rarr; ~1,140&nbsp;MB, 1.43&times; whole-store</b>. On a
+1024-d store, where vectors are ~90% of the file, it lands nearer 1.6&times;.
+
+<b>The key must be the embedding blob, not the text &mdash; and this is the constraint that decides
+the design.</b> <code>export_shard</code> emits <code>name/tap/path/kind/cix/snip/digest/embedding</code>
+and <code>snip</code> is <code>text[:200]</code>: <b>the full chunk text is not in the store and not
+in the shard</b>, so <code>import_shard</code> has nothing to hash. A <code>text_hash</code> column
+is unpopulatable on the import path. <code>sha256(embedding)</code> is equivalent by construction
+&mdash; identical texts are embedded once and serialize to identical bytes &mdash; and it works for
+imports and for the migration.
+
+<b>No join table.</b> <code>#544</code> proposed a chunk&rarr;tap join table; <code>chunks</code>
+already <em>is</em> that table and already carries <code>tap</code>. The change is one
+<code>vid</code> column on <code>chunks</code>, a <code>vectors</code> relation, and
+<code>vec_chunks_bin</code> keyed by <code>vid</code>.
+
+<b>Three things break, and one of them is an ordering.</b> <code>_delete_matching</code> drops
+vectors <em>then</em> rows; refcounted GC must run <em>after</em> the row delete or it counts
+references about to vanish. <code>_store_vector</code>'s orphan guard stops being sufficient, because
+under dedup a <code>_COMMIT_EVERY</code> interrupt makes orphans structural rather than exceptional.
+And <code>export_shard</code>'s <code>v.id = c.id</code> join becomes <code>v.vid = c.vid</code>
+&mdash; the same shape whose breakage the docstring already records as having killed two scheduled
+<code>shards</code> runs.
+
+<b>In place, not an <code>INDEX_VERSION</code> bump.</b> Every input is already on disk, so the
+migration re-encodes rather than re-embeds &mdash; the posture <code>quantize()</code> established,
+and roughly its cost (~1360&nbsp;s, ~2&times; peak while both copies coexist), except that this one
+ends <em>smaller</em>. <code>dense.py</code> promises in writing that v3's re-embed is
+&ldquo;the last one this mechanism needs&rdquo;; a v4 wipe would charge every user a full re-embed
+to reclaim disk, which is the wrong trade even where the embedder is free.
+
+<b>The quality half is already fixed, and this card is not blocked on it.</b>
+<code>MAX_PER_VECTOR</code> thins the candidate pool and caps the page, which is what stopped one
+1,464-copy cluster taking 60 of 60 result slots. Dedup would make the pool distinct
+<em>structurally</em> rather than by thinning, and would let <code>_knn</code> stop hashing 2,048
+blobs per query &mdash; a simplification, not a rescue.
+
+<b>The free bonus nobody has claimed:</b> <code>seen</code> in <code>_embed_and_store</code> is a
+local rebuilt per call, so reuse is <em>per build</em>. A persisted blob-keyed table makes it
+cross-build and cross-tap: a newly tapped mirror registry would cost zero embeddings.
+
+<b>Biggest risk, named:</b> a third schema state. The store already branches float32 vs quantized
+through every vector-touching function; &ldquo;deduplicated or not&rdquo; makes four combinations,
+each needing mutant-killing tests. And the one invariant that catches silent corruption &mdash;
+<code>test_dense_entry_reuse.py</code>'s two-way <code>chunks.id</code> &harr; <code>vec_raw.id</code>
+bijection &mdash; must be <b>rewritten</b> rather than extended, because dedup breaks one direction
+by design. Its replacement: every <code>chunks.vid</code> resolves, and every <code>vectors</code>
+row has at least one referent.

{boost_skill_cli-1.2.61 → boost_skill_cli-1.2.63}/docs/roadmap.html

@@ -133,7 +133,7 @@
       <div class="stat"><b>20</b><span>Shipped</span></div>
       <div class="stat"><b>1</b><span>Next up</span></div>
       <div class="stat"><b>7</b><span>Planned</span></div>
-      <div class="stat"><b>290</b><span>Loop finds</span></div>
+      <div class="stat"><b>291</b><span>Loop finds</span></div>
       <!-- /ROADMAP:stats -->
     </div>
   </section>
@@ -4886,6 +4886,81 @@ existing table, and <code>build</code> already wipes on a version change. One fu
           <span class="m">one changed file re-embeds 29 min for 1.9 s of new text</span>
         </div>
       </article>
+
+      <article class="cap rcard" id="dense-store-holds-every-vector-once-per-copy">
+        <div class="head"><span class="pill planned">Planned</span><span class="cat">Search · Storage</span></div>
+        <h3>The dense store keeps one vector per <em>copy</em>, not one per distinct text</h3>
+        <p><code>_embed_and_store</code> already buys each distinct text <b>once</b> &mdash; <code>seen</code>
+de-duplicates before the provider call, which is why <code>build</code>'s progress total counts
+distinct texts rather than rows. The <em>storage</em> never got the same treatment: every chunk row
+still gets its own <code>vec_raw</code> blob and its own <code>vec_chunks_bin</code> row, so a text
+vendored into 1,464 skills is stored 1,464 times.
+
+<b>Measured on a real store</b> (657,587 chunks, 384-d <code>BAAI/bge-small-en-v1.5</code>):
+657,587 vector rows collapse to <b>396,638 distinct</b> &mdash; 39.7% repeats, <b>1.658&times;</b>.
+The earlier 750,416-chunk / 1024-d Voyage store measured 42.9%, so the ratio is a property of the
+corpus rather than of one embedder.
+
+<b>State the payoff in the right scope.</b> The <code>#544</code> note said &ldquo;~1.75&times;&rdquo;
+without saying of what, and that reads as a whole-store number. It is not. From <code>dbstat</code>
+on the live file: <code>vec_raw</code> 1,287.6&nbsp;MB and <code>vec_chunks_bin</code> 45.1&nbsp;MB
+of a 1,634&nbsp;MB total, so vectors are <b>81.6%</b> of the file. Deduplicating them saves
+<b>529&nbsp;MB</b>; add back a <code>vid</code> column, a hash index and a <code>chunks(vid)</code>
+index and the honest figure is <b>1,634 &rarr; ~1,140&nbsp;MB, 1.43&times; whole-store</b>. On a
+1024-d store, where vectors are ~90% of the file, it lands nearer 1.6&times;.
+
+<b>The key must be the embedding blob, not the text &mdash; and this is the constraint that decides
+the design.</b> <code>export_shard</code> emits <code>name/tap/path/kind/cix/snip/digest/embedding</code>
+and <code>snip</code> is <code>text[:200]</code>: <b>the full chunk text is not in the store and not
+in the shard</b>, so <code>import_shard</code> has nothing to hash. A <code>text_hash</code> column
+is unpopulatable on the import path. <code>sha256(embedding)</code> is equivalent by construction
+&mdash; identical texts are embedded once and serialize to identical bytes &mdash; and it works for
+imports and for the migration.
+
+<b>No join table.</b> <code>#544</code> proposed a chunk&rarr;tap join table; <code>chunks</code>
+already <em>is</em> that table and already carries <code>tap</code>. The change is one
+<code>vid</code> column on <code>chunks</code>, a <code>vectors</code> relation, and
+<code>vec_chunks_bin</code> keyed by <code>vid</code>.
+
+<b>Three things break, and one of them is an ordering.</b> <code>_delete_matching</code> drops
+vectors <em>then</em> rows; refcounted GC must run <em>after</em> the row delete or it counts
+references about to vanish. <code>_store_vector</code>'s orphan guard stops being sufficient, because
+under dedup a <code>_COMMIT_EVERY</code> interrupt makes orphans structural rather than exceptional.
+And <code>export_shard</code>'s <code>v.id = c.id</code> join becomes <code>v.vid = c.vid</code>
+&mdash; the same shape whose breakage the docstring already records as having killed two scheduled
+<code>shards</code> runs.
+
+<b>In place, not an <code>INDEX_VERSION</code> bump.</b> Every input is already on disk, so the
+migration re-encodes rather than re-embeds &mdash; the posture <code>quantize()</code> established,
+and roughly its cost (~1360&nbsp;s, ~2&times; peak while both copies coexist), except that this one
+ends <em>smaller</em>. <code>dense.py</code> promises in writing that v3's re-embed is
+&ldquo;the last one this mechanism needs&rdquo;; a v4 wipe would charge every user a full re-embed
+to reclaim disk, which is the wrong trade even where the embedder is free.
+
+<b>The quality half is already fixed, and this card is not blocked on it.</b>
+<code>MAX_PER_VECTOR</code> thins the candidate pool and caps the page, which is what stopped one
+1,464-copy cluster taking 60 of 60 result slots. Dedup would make the pool distinct
+<em>structurally</em> rather than by thinning, and would let <code>_knn</code> stop hashing 2,048
+blobs per query &mdash; a simplification, not a rescue.
+
+<b>The free bonus nobody has claimed:</b> <code>seen</code> in <code>_embed_and_store</code> is a
+local rebuilt per call, so reuse is <em>per build</em>. A persisted blob-keyed table makes it
+cross-build and cross-tap: a newly tapped mirror registry would cost zero embeddings.
+
+<b>Biggest risk, named:</b> a third schema state. The store already branches float32 vs quantized
+through every vector-touching function; &ldquo;deduplicated or not&rdquo; makes four combinations,
+each needing mutant-killing tests. And the one invariant that catches silent corruption &mdash;
+<code>test_dense_entry_reuse.py</code>'s two-way <code>chunks.id</code> &harr; <code>vec_raw.id</code>
+bijection &mdash; must be <b>rewritten</b> rather than extended, because dedup breaks one direction
+by design. Its replacement: every <code>chunks.vid</code> resolves, and every <code>vectors</code>
+row has at least one referent.</p>
+        <div class="meta">
+          <span class="m">Complexity <b>L</b></span>
+          <span class="m">Impact <b>Med</b></span>
+          <span class="m">Wow <b class="wow">★★★</b></span>
+          <span class="m">39.7% of vector rows are byte-identical repeats; 1.43x whole-store, no re-embedding</span>
+        </div>
+      </article>
       <!-- /ROADMAP:cards -->
 
     </div>