boost-skill-cli 1.0.322__tar.gz → 1.0.324__tar.gz

{boost_skill_cli-1.0.322/boost_skill_cli.egg-info → boost_skill_cli-1.0.324}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: boost-skill-cli
-Version: 1.0.322
+Version: 1.0.324
 Summary: boost — Homebrew for AI coding skills
 Author: Jonathan Reyes
 License: GNU GENERAL PUBLIC LICENSE

{boost_skill_cli-1.0.322 → boost_skill_cli-1.0.324}/boost_cli/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '1.0.322'
-__version_tuple__ = version_tuple = (1, 0, 322)
+__version__ = version = '1.0.324'
+__version_tuple__ = version_tuple = (1, 0, 324)
 
 __commit_id__ = commit_id = None

{boost_skill_cli-1.0.322 → boost_skill_cli-1.0.324}/boost_cli/core/rag.py

@@ -15,13 +15,16 @@ no AI -> the BM25 order is returned as-is.
 """
 from __future__ import annotations
 
+import contextlib
+import hashlib
 import json
 import math
 import re
+import sqlite3
 from collections import defaultdict
 from collections.abc import Sequence
 from pathlib import Path
-from typing import Dict, List, Optional, Tuple
+from typing import Dict, List, Optional, Tuple, cast
 
 try:  # TypedDict lives in typing on 3.9+, kept optional for safety
     from typing import TypedDict
@@ -33,7 +36,7 @@ from . import ai, catalog, frontmatter, gitutil, paths, registry, util
 # v2: `snip` stores a larger head of the matched chunk (not just SNIP_WIDTH
 # chars) so `retrieve` can window it onto the query terms; bump forces a
 # one-time reindex.
-INDEX_VERSION = 4
+INDEX_VERSION = 6
 ENGINE = "bm25"
 
 # Chunking defaults (documented in docs/rag-architecture.md §4).
@@ -62,10 +65,13 @@ _STOPWORDS = frozenset({
 })
 
 
-class Hit(TypedDict):  # type: ignore[misc]
+class Hit(TypedDict, total=False):  # type: ignore[misc]
     entry: dict     # the live catalog entry (name, tap, kind, skill_md, ...)
     score: float    # BM25 relevance (higher = better)
     snippet: str    # best-matching passage, for display + rerank context
+    content: str    # digest of the body, for collapsing byte-identical copies.
+                    # Optional: dense hits arrive without one and have it
+                    # attached from the BM25 index (see `_with_content`).
 
 
 # --------------------------------------------------------------- tokenizing
@@ -193,13 +199,16 @@ def _make_docs(entries: List[dict], tap_paths: Dict[str, Path]) -> List[dict]:
     docs: List[dict] = []
     for e in entries:
         body = read_body(e, tap_paths)
+        # Hashed here because the body is already in hand; doing it at query
+        # time would mean re-reading ~30k files to answer one search.
+        digest = hashlib.sha256(body.strip().encode("utf-8", "replace")).hexdigest()[:16]
         tf: Dict[str, int] = defaultdict(int)
         for tok in tokenize(surface(e) + "\n" + body):
             tf[tok] += 1
         if not tf:
             continue
         docs.append({
-            "n": e["name"], "t": e["tap"], "f": e["skill_md"],
+            "n": e["name"], "t": e["tap"], "f": e["skill_md"], "h": digest,
             "k": e.get("kind", "skill"),
             "c": 0, "l": sum(tf.values()),
             "snip": body[:SNIP_STORE].strip(),  # windowed at retrieve time
@@ -211,7 +220,7 @@ def _make_docs(entries: List[dict], tap_paths: Dict[str, Path]) -> List[dict]:
 def _postings_to_doc_tf(raw: dict) -> Dict[int, Dict[str, int]]:
     """Invert persisted postings back to per-doc term frequencies."""
     doc_tf: Dict[int, Dict[str, int]] = defaultdict(dict)
-    for term, plist in raw.get("postings", {}).items():
+    for term, plist in (raw.get("postings") or _all_postings()).items():
         for doc_id, tf in plist:
             doc_tf[doc_id][term] = tf
     return doc_tf
@@ -259,6 +268,94 @@ def build(entries: Optional[List[dict]] = None, force: bool = False) -> dict:
     }
 
 
+def postings_path() -> Path:
+    return paths.cache_dir() / "rag_postings.sqlite"
+
+
+def _write_postings(postings: Dict[str, List[List[int]]]) -> None:
+    """Persist the term -> [(doc, tf)] map to SQLite, replacing any existing one.
+
+    Postings were the largest part of the JSON index — measured 64% of it after
+    unchunking — and `json.loads` had to materialise every one of them to answer
+    a query that touches a handful of terms. On a real 83-tap install that was
+    8-13 s and multiple GB resident per cold `boost search`, against 31-70 ms of
+    actual scoring.
+
+    SQLite is used purely as an indexed key-value store here, not as a search
+    engine: BM25 scoring stays in `_bm25` byte for byte, so this changes cold
+    start cost and nothing about which results come back.
+    """
+    paths.ensure_dirs()
+    final = postings_path()
+    tmp = final.with_suffix(".sqlite.tmp")
+    with contextlib.suppress(OSError):
+        tmp.unlink()
+    con = sqlite3.connect(str(tmp))
+    try:
+        # No durability needed: this file is a derived cache, and a crash
+        # mid-build leaves the .tmp behind rather than a torn index.
+        con.execute("PRAGMA journal_mode=OFF")
+        con.execute("PRAGMA synchronous=OFF")
+        con.execute("CREATE TABLE postings (term TEXT, doc INTEGER, tf INTEGER)")
+        con.executemany(
+            "INSERT INTO postings (term, doc, tf) VALUES (?, ?, ?)",
+            ((term, doc, tf) for term, plist in postings.items()
+             for doc, tf in plist))
+        # Built after the insert: maintaining it per-row is far slower.
+        con.execute("CREATE INDEX postings_term ON postings(term)")
+        con.commit()
+    finally:
+        con.close()
+    tmp.replace(final)          # atomic swap, same as the JSON half
+
+
+def read_postings(terms: Sequence[str]) -> Dict[str, List[List[int]]]:
+    """Postings for just these terms — the whole point of the SQLite store.
+
+    A query touches a handful of terms out of tens of thousands, so this reads
+    kilobytes where the JSON index read the entire file.
+    """
+    uniq = sorted(set(terms))
+    if not uniq:
+        return {}
+    try:
+        con = sqlite3.connect("file:%s?mode=ro" % postings_path(), uri=True)
+    except sqlite3.Error:
+        return {}
+    out: Dict[str, List[List[int]]] = defaultdict(list)
+    try:
+        # Chunked: SQLite caps host parameters (999 on older builds), and a
+        # long query can exceed it.
+        for i in range(0, len(uniq), 500):
+            batch = uniq[i:i + 500]
+            q = ("SELECT term, doc, tf FROM postings WHERE term IN (%s)"  # noqa: S608  interpolates only `?` placeholders; terms are bound params
+                 % ",".join("?" * len(batch)))
+            for term, doc, tf in con.execute(q, batch):
+                out[term].append([doc, tf])
+    except sqlite3.Error:
+        return {}
+    finally:
+        con.close()
+    return dict(out)  # noqa: FURB123  out is a defaultdict; .copy() would keep the factory
+
+
+def _all_postings() -> Dict[str, List[List[int]]]:
+    """Every posting — only the incremental rebuild path needs this."""
+    try:
+        con = sqlite3.connect("file:%s?mode=ro" % postings_path(), uri=True)
+    except sqlite3.Error:
+        return {}
+    out: Dict[str, List[List[int]]] = defaultdict(list)
+    try:
+        for term, doc, tf in con.execute("SELECT term, doc, tf FROM postings"):
+            out[term].append([doc, tf])
+    except sqlite3.Error:
+        return {}
+    finally:
+        con.close()
+    return dict(out)  # noqa: FURB123  out is a defaultdict; .copy() would keep the factory
+
+
 def _save(docs: List[dict], commits: Dict[str, str]) -> None:
     postings: Dict[str, List[List[int]]] = defaultdict(list)
     meta_docs: List[dict] = []
@@ -268,6 +365,7 @@ def _save(docs: List[dict], commits: Dict[str, str]) -> None:
             postings[term].append([doc_id, tf])
         total_len += d["l"]
         meta_docs.append({"n": d["n"], "t": d["t"], "f": d["f"], "k": d["k"],
+                          "h": d.get("h", ""),
                           "c": d["c"], "l": d["l"], "snip": d["snip"]})
     payload = {
         "version": INDEX_VERSION,
@@ -279,12 +377,12 @@ def _save(docs: List[dict], commits: Dict[str, str]) -> None:
         "stats": {"docs": len(docs),
                   "avg_len": (total_len / len(docs)) if docs else 0.0},
         "docs": meta_docs,
-        "postings": dict(postings.items()),
     }
     paths.ensure_dirs()
     p = index_path()
     # Atomic swap: a bare write_text here can leave a truncated/corrupt index
     # on disk if the process dies mid-write or a query reads concurrently.
+    _write_postings(postings)
     util.atomic_write_text(p, json.dumps(payload))
     # Drop the mtime-keyed cache so a reindex is visible to an immediately
     # following query even when the filesystem mtime granularity is coarse.
@@ -322,9 +420,14 @@ def _load_raw() -> Optional[dict]:
 
 
 def ready() -> bool:
-    """True when a usable index exists on disk."""
+    """True when a usable index exists on disk.
+
+    Both halves are required: the JSON carries the documents, the SQLite file
+    carries the postings, and scoring needs both. A docs-only index reports not
+    ready rather than silently returning zero hits for every query.
+    """
     raw = _load_raw()
-    return bool(raw and raw.get("docs"))
+    return bool(raw and raw.get("docs") and postings_path().exists())
 
 
 def ensure() -> bool:
@@ -346,11 +449,17 @@ def ensure() -> bool:
     return ready()
 
 
-def _bm25(query_terms: List[str], raw: dict) -> Dict[int, float]:
+def _bm25(query_terms: List[str], raw: dict,
+          postings: Dict[str, List[List[int]]]) -> Dict[int, float]:
+    """Score documents for ``query_terms``. Pure: postings are passed in.
+
+    Reading them inside would hide I/O in the one function whose arithmetic is
+    pinned exactly (`TestBm25Math`), and would tie the scorer to where postings
+    happen to live — which is the thing this change moved.
+    """
     docs = raw["docs"]
     n = len(docs)
     avg = raw.get("stats", {}).get("avg_len") or 1.0
-    postings = raw["postings"]
     scores: Dict[int, float] = defaultdict(float)
     for term in set(query_terms):
         plist = postings.get(term)
@@ -418,6 +527,54 @@ def entry_key(entry: dict) -> Tuple[str, str]:
     return (entry["tap"], entry["skill_md"])
 
 
+def dedupe_by_content(hits: List[Hit], limit: int) -> List[Hit]:
+    """Collapse byte-identical copies, then take ``limit``.
+
+    Measured over 77 tapped registries: 78.3% of 29,938 entries are
+    byte-identical duplicates (largest cluster 40 copies), and the
+    natural-language query set averaged 4.94 of 10 result slots consumed by a
+    repeat — worst case 9 of 10. Registries mirror each other, so N copies of one
+    rule across N taps ate N of the ten slots a user sees.
+
+    Clustering is on the **body**, never the name. 82.7% of entries share a name
+    but only 78.3% share a body, and that ~4.4% gap is real distinct content —
+    the ``admin-interface-rule`` pair fixed in #366 is two different rules under
+    one name. Clustering on content cannot make that mistake: of 14,153 distinct
+    bodies, the number of clusters spanning more than one name is zero.
+
+    Within a cluster every copy is byte-identical, so *which* one survives is
+    purely about where the user would rather install from: a ``curated`` tap wins
+    over a better-ranked uncurated one. The cluster keeps its best score either
+    way, so promoting the trusted copy never demotes the result.
+
+    A hit with no recorded content hash is always kept — two unknowns must not
+    collapse into each other, which would hide real results.
+    """
+    best: Dict[str, int] = {}          # content hash -> index into `out`
+    out: List[Hit] = []
+    for hit in hits:
+        digest = hit.get("content")
+        if not digest:
+            out.append(hit)            # unknown content is never a match
+            continue
+        seen = best.get(digest)
+        if seen is None:
+            best[digest] = len(out)
+            out.append(hit)
+            continue
+        # Same body already shown. Keep its rank, but prefer a curated source.
+        kept = out[seen]
+        if hit["entry"].get("curated") and not kept["entry"].get("curated"):
+            # cast, not a literal annotation: `Hit` is total=False for the
+            # optional `content` key, and NotRequired needs 3.11 while boost
+            # targets 3.9.
+            out[seen] = cast(Hit, {"entry": hit["entry"],
+                                   "score": kept["score"],
+                                   "snippet": kept["snippet"],
+                                   "content": digest})
+    return out[:limit]
+
+
 def retrieve(query: str, k: int = 60, kind: Optional[str] = None,
              entries: Optional[List[dict]] = None) -> List[Hit]:
     """Top-k full-content hits for ``query``. ``[]`` if the index is empty."""
@@ -430,8 +587,8 @@ def retrieve(query: str, k: int = 60, kind: Optional[str] = None,
     entries = catalog.all_entries() if entries is None else entries
     live = {entry_key(e): e for e in entries}
     docs = raw["docs"]
-    best: Dict[Tuple[str, str], Tuple[float, str]] = {}
-    for doc_id, score in _bm25(terms, raw).items():
+    best: Dict[Tuple[str, str], Tuple[float, str, str]] = {}
+    for doc_id, score in _bm25(terms, raw, read_postings(terms)).items():
         d = docs[doc_id]
         if kind is not None and d["k"] != kind:
             continue
@@ -440,17 +597,20 @@ def retrieve(query: str, k: int = 60, kind: Optional[str] = None,
             continue
         prev = best.get(key)
         if prev is None or score > prev[0]:
-            best[key] = (score, d["snip"])
+            best[key] = (score, d["snip"], d.get("h", ""))
     # Tie-break on the displayed name, not the identity key: the key is now
     # (tap, path), and ordering by it would silently reshuffle equal-scoring
     # results relative to every previous release.
     ranked = sorted(best.items(),
                     key=lambda kv: (-kv[1][0], live[kv[0]]["name"], kv[0]))
+    # Over-fetch, then collapse byte-identical copies before taking k — the
+    # duplicates have to be removed from the pool, not from the visible page,
+    # or they still consume the slots they were removed from.
     hits: List[Hit] = [
-        {"entry": live[key], "score": score,
+        {"entry": live[key], "score": score, "content": digest,
          "snippet": _passage(snip, terms)}  # type: ignore[typeddict-item]
-        for key, (score, snip) in ranked[:k]]
-    return hits
+        for key, (score, snip, digest) in ranked]
+    return dedupe_by_content(hits, k)
 
 
 def rerank(query: str, hits: List[Hit], limit: int = 10) -> Tuple[List[Hit], str]:
@@ -516,6 +676,39 @@ def rrf_fuse(rankings: List[List[Hit]], limit: int = 60) -> List[Hit]:
     return fused
 
 
+def content_hashes() -> Dict[Tuple[str, str], str]:
+    """``entry_key`` -> content hash, read from the BM25 index.
+
+    The dense store does not carry the hash and does not need to: BM25 indexes
+    the same entries and auto-builds on first search, so one map serves every
+    engine. Keeping it in one place also means the two engines can never
+    disagree about which copies are identical.
+    """
+    raw = _load_raw()
+    if not raw:
+        return {}
+    return {(d["t"], d["f"]): d["h"]
+            for d in raw.get("docs", []) if d.get("h")}
+
+
+def _with_content(hits: List[Hit]) -> List[Hit]:
+    """Attach content hashes to hits that lack them (dense, and fused output)."""
+    if all(h.get("content") for h in hits):
+        return hits
+    hashes = content_hashes()
+    if not hashes:
+        return hits
+    filled: List[Hit] = []
+    for h in hits:
+        if h.get("content"):
+            filled.append(h)
+            continue
+        filled.append(cast(Hit, {
+            "entry": h["entry"], "score": h["score"], "snippet": h["snippet"],
+            "content": hashes.get(entry_key(h["entry"]), "")}))
+    return filled
+
+
 def retrieve_any(query: str, k: int = 60, kind: Optional[str] = None,
                  entries: Optional[List[dict]] = None
                  ) -> Tuple[Optional[List[Hit]], str]:
@@ -551,13 +744,18 @@ def retrieve_any(query: str, k: int = 60, kind: Optional[str] = None,
     # counts as a ranking to fuse.
     bm25_hits = retrieve(query, k=pool, kind=kind,
                          entries=entries) if ready() else None
+    # Dedupe at this seam as well as inside `retrieve`, because dense hits carry
+    # no hash of their own and fusion can reintroduce a duplicate that BM25
+    # already dropped — the copies are distinct `(tap, path)` keys, so RRF has
+    # no reason to treat them as one.
     if dense_hits and bm25_hits:
-        return (rrf_fuse([bm25_hits, dense_hits], limit=k),
+        fused = rrf_fuse([bm25_hits, dense_hits], limit=max(k, RRF_K))
+        return (dedupe_by_content(_with_content(fused), k),
                 "hybrid RRF (BM25 + dense)")
     if dense_hits:
-        return dense_hits[:k], "dense vectors"
+        return dedupe_by_content(_with_content(dense_hits), k), "dense vectors"
     if bm25_hits is not None:
-        return bm25_hits[:k], "BM25 full-content"
+        return bm25_hits[:k], "BM25 full-content"     # already deduped
     return None, ""
 
 

{boost_skill_cli-1.0.322 → boost_skill_cli-1.0.324/boost_skill_cli.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: boost-skill-cli
-Version: 1.0.322
+Version: 1.0.324
 Summary: boost — Homebrew for AI coding skills
 Author: Jonathan Reyes
 License: GNU GENERAL PUBLIC LICENSE

{boost_skill_cli-1.0.322 → boost_skill_cli-1.0.324}/boost_skill_cli.egg-info/SOURCES.txt

@@ -483,6 +483,7 @@ tests/unit/test_codeql_analysis_key.py
 tests/unit/test_command_reference_fresh.py
 tests/unit/test_community_health_files.py
 tests/unit/test_config.py
+tests/unit/test_content_dedupe.py
 tests/unit/test_dense.py
 tests/unit/test_dense_fallback.py
 tests/unit/test_dense_fix_hint.py

{boost_skill_cli-1.0.322 → boost_skill_cli-1.0.324}/docs/roadmap/items/bm25-index-is-one-json-blob.md

@@ -9,7 +9,7 @@ impact: High
 wow: 5
 note: 2.5 GB RSS per search at 50k items
 order: 72
-owner: loop/unchunked-bm25-index
+owner: loop/postings-out-of-json
 pr:
 title: The BM25 index is one JSON blob, and it stops working between 10k and 50k items
 ---
@@ -99,3 +99,34 @@ memory but not the scan, and in the common case (one tap of many changed) nearly
 kept, so it saves little of that either. Denormalising <code>tf</code> back into the documents would
 fix it directly but roughly duplicates the 64% that postings already occupy. The real fix is the
 storage format, which is win (2) &mdash; worth treating them as one item rather than two.
+
+<b>Win (2) is shipped: postings now live in SQLite, and a query reads its terms rather than the whole
+index.</b> Measured on the pinned corpus &mdash; JSON 2.05 MB &rarr; <b>0.76 MB</b>, load
+0.015 s &rarr; <b>0.0013 s</b>, cold retrieve <b>0.0003 s</b>.
+
+<b>SQLite as a key-value store, not as FTS5.</b> The card offered FTS5 or a binary format, and FTS5
+was the tempting one &mdash; it is compiled in, verified on CPython 3.14 (SQLite 3.53.4) and system
+3.9 (3.54.0). But FTS5 replaces BM25 with <em>its own</em> ranking: different tokenizer, different
+parameters, different results. The measured problem is cold-start cost, not scoring quality, so
+adopting FTS5 would risk what works to fix what does not. Here <code>_bm25</code> is unchanged byte
+for byte, and the eval gate proves the change is behaviour-neutral: 0.978 / 0.791 / 0.854 / 0.882,
+identical before and after. <code>_bm25</code> also now takes postings as an <em>argument</em>, so
+the one function whose arithmetic is pinned exactly contains no I/O and is not coupled to where
+postings are stored.
+
+<b>The honest cost: total disk goes up.</b> The store is 3.89 MB against 1.31 MB of JSON postings,
+because SQLite keeps the term string on every row plus an index over it. Total on-disk is 4.65 MB
+versus 2.05 MB. This card says <b>&ldquo;target RSS, not bytes&rdquo;</b> and that is why the trade
+is the right way round &mdash; the hot path now reads kilobytes instead of materialising megabytes of
+Python objects &mdash; but it is a real cost, not a free win. Interning terms into a
+<code>term_id</code> table would recover most of it and is the obvious follow-up if the size matters.
+
+<b>Both halves are now required.</b> <code>ready()</code> checks for the postings store as well as
+the documents, so a docs-only index reports not-ready instead of silently scoring every query to
+zero hits. <code>INDEX_VERSION</code> 5 &rarr; 6 forces the one-time rebuild.
+
+<b>Still open:</b> <code>snip</code> text off the hot path &mdash; now only 14% of the JSON half, so
+much smaller than this card assumed &mdash; and the O(entire-corpus) incremental reindex. The latter
+is closer than it was: per-document term frequencies are now queryable by document rather than only
+recoverable by inverting every posting, so scoping the rebuild to the changed taps is a tractable
+follow-up rather than a format change.

{boost_skill_cli-1.0.322 → boost_skill_cli-1.0.324}/docs/roadmap/items/near-duplicate-items-eat-the-result-slots.md

@@ -9,7 +9,7 @@ impact: High
 wow: 4
 note: 56.3% of entries share a name
 order: 71
-owner: loop/entry-key-collision
+owner: loop/content-hash-dedup
 pr:
 title: Near-duplicate items consume the top-10, and it gets worse with every tap
 ---
@@ -123,3 +123,39 @@ genuinely differ (the <code>admin-interface-rule</code> pair fixed in <code>#366
 here as duplicate slots but must <em>not</em> be merged. That is precisely why the card asks for
 clustering on content hash rather than on name, and this curve is an upper bound on what dedup could
 reclaim, not a target.
+
+<b>Shipped: content-hash dedup, measured 4.94 &rarr; 0.60 duplicate slots per query</b> over the
+77-tap corpus (worst case 9 of 10 &rarr; 4). The residual 0.60 is correct rather than leftover: those
+are entries sharing a <em>name</em> whose bodies genuinely differ, which must stay separate.
+
+Two measurements settled the design the card left open (&ldquo;content hash <em>or</em> canonical
+name&rdquo;). Of 29,938 entries, <b>78.3% are byte-identical duplicates</b> &mdash; 14,153 distinct
+bodies, largest cluster <b>40 copies</b> &mdash; while 82.7% share a name. That ~4.4% gap is real
+distinct content, so name clustering would merge exactly the pairs <code>#366</code> proved must not
+be merged. And clustering on content cannot make the opposite mistake here: the number of content
+clusters spanning more than one name is <b>0</b>, so collapsing by body never merges two
+differently-named items. Content hashing is strictly the safer of the two options the card offered,
+which was not obvious before measuring.
+
+The quality prior is <code>curated</code>: inside a cluster every copy is byte-identical, so the one
+worth surfacing is the one from a tap the user marked trusted. The cluster keeps its best score when
+that swap happens, so promoting the trusted copy never demotes the result.
+
+Mechanically, the body hash is computed in <code>_make_docs</code> where the body is already read
+&mdash; doing it at query time would mean re-reading ~30k files to answer one search &mdash; and
+persisted per document (<code>INDEX_VERSION</code> 4 &rarr; 5). Dense hits carry no hash of their own
+and do not need a second schema: <code>content_hashes()</code> serves one map to every engine from
+the BM25 index, so the engines can never disagree about which copies are identical. Dedup runs inside
+<code>retrieve</code> <em>and</em> at the <code>retrieve_any</code> seam, because fusion can
+reintroduce a copy BM25 already dropped &mdash; the copies are distinct <code>(tap, path)</code>
+keys, so RRF has no reason to treat them as one. Collapsing happens before <code>k</code> is applied,
+or the duplicates would still consume the slots they were removed from.
+
+Cost on the pinned eval corpus: <b>none</b> &mdash; 0.978 / 0.791 / 0.854 / 0.882, identical to
+before, all four floors passing. That corpus has little content duplication, which is exactly why the
+83-tap measurement had to exist.
+
+<b>Still open:</b> a richer quality prior than <code>curated</code> (the card names stars, recency,
+maintenance; only <code>curated</code> is available on an entry today), and
+<code>core/typosquat.py</code>'s confusion machinery for near-identical rather than byte-identical
+bodies. Both are refinements &mdash; the byte-identical case is 78.3% of the problem.

{boost_skill_cli-1.0.322 → boost_skill_cli-1.0.324}/docs/roadmap.html

@@ -3472,7 +3472,43 @@ symptom the card names, not content-identical bodies — two rules that shar
 genuinely differ (the <code>admin-interface-rule</code> pair fixed in <code>#366</code>) are counted
 here as duplicate slots but must <em>not</em> be merged. That is precisely why the card asks for
 clustering on content hash rather than on name, and this curve is an upper bound on what dedup could
-reclaim, not a target.</p>
+reclaim, not a target.
+
+<b>Shipped: content-hash dedup, measured 4.94 &rarr; 0.60 duplicate slots per query</b> over the
+77-tap corpus (worst case 9 of 10 &rarr; 4). The residual 0.60 is correct rather than leftover: those
+are entries sharing a <em>name</em> whose bodies genuinely differ, which must stay separate.
+
+Two measurements settled the design the card left open (&ldquo;content hash <em>or</em> canonical
+name&rdquo;). Of 29,938 entries, <b>78.3% are byte-identical duplicates</b> &mdash; 14,153 distinct
+bodies, largest cluster <b>40 copies</b> &mdash; while 82.7% share a name. That ~4.4% gap is real
+distinct content, so name clustering would merge exactly the pairs <code>#366</code> proved must not
+be merged. And clustering on content cannot make the opposite mistake here: the number of content
+clusters spanning more than one name is <b>0</b>, so collapsing by body never merges two
+differently-named items. Content hashing is strictly the safer of the two options the card offered,
+which was not obvious before measuring.
+
+The quality prior is <code>curated</code>: inside a cluster every copy is byte-identical, so the one
+worth surfacing is the one from a tap the user marked trusted. The cluster keeps its best score when
+that swap happens, so promoting the trusted copy never demotes the result.
+
+Mechanically, the body hash is computed in <code>_make_docs</code> where the body is already read
+&mdash; doing it at query time would mean re-reading ~30k files to answer one search &mdash; and
+persisted per document (<code>INDEX_VERSION</code> 4 &rarr; 5). Dense hits carry no hash of their own
+and do not need a second schema: <code>content_hashes()</code> serves one map to every engine from
+the BM25 index, so the engines can never disagree about which copies are identical. Dedup runs inside
+<code>retrieve</code> <em>and</em> at the <code>retrieve_any</code> seam, because fusion can
+reintroduce a copy BM25 already dropped &mdash; the copies are distinct <code>(tap, path)</code>
+keys, so RRF has no reason to treat them as one. Collapsing happens before <code>k</code> is applied,
+or the duplicates would still consume the slots they were removed from.
+
+Cost on the pinned eval corpus: <b>none</b> &mdash; 0.978 / 0.791 / 0.854 / 0.882, identical to
+before, all four floors passing. That corpus has little content duplication, which is exactly why the
+83-tap measurement had to exist.
+
+<b>Still open:</b> a richer quality prior than <code>curated</code> (the card names stars, recency,
+maintenance; only <code>curated</code> is available on an entry today), and
+<code>core/typosquat.py</code>'s confusion machinery for near-identical rather than byte-identical
+bodies. Both are refinements &mdash; the byte-identical case is 78.3% of the problem.</p>
         <div class="meta">
           <span class="m">Complexity <b>M</b></span>
           <span class="m hi">Impact <b>High</b></span>
@@ -3569,7 +3605,38 @@ taps means scanning every posting whatever the filter. Narrowing the inversion t
 memory but not the scan, and in the common case (one tap of many changed) nearly every document is
 kept, so it saves little of that either. Denormalising <code>tf</code> back into the documents would
 fix it directly but roughly duplicates the 64% that postings already occupy. The real fix is the
-storage format, which is win (2) &mdash; worth treating them as one item rather than two.</p>
+storage format, which is win (2) &mdash; worth treating them as one item rather than two.
+
+<b>Win (2) is shipped: postings now live in SQLite, and a query reads its terms rather than the whole
+index.</b> Measured on the pinned corpus &mdash; JSON 2.05 MB &rarr; <b>0.76 MB</b>, load
+0.015 s &rarr; <b>0.0013 s</b>, cold retrieve <b>0.0003 s</b>.
+
+<b>SQLite as a key-value store, not as FTS5.</b> The card offered FTS5 or a binary format, and FTS5
+was the tempting one &mdash; it is compiled in, verified on CPython 3.14 (SQLite 3.53.4) and system
+3.9 (3.54.0). But FTS5 replaces BM25 with <em>its own</em> ranking: different tokenizer, different
+parameters, different results. The measured problem is cold-start cost, not scoring quality, so
+adopting FTS5 would risk what works to fix what does not. Here <code>_bm25</code> is unchanged byte
+for byte, and the eval gate proves the change is behaviour-neutral: 0.978 / 0.791 / 0.854 / 0.882,
+identical before and after. <code>_bm25</code> also now takes postings as an <em>argument</em>, so
+the one function whose arithmetic is pinned exactly contains no I/O and is not coupled to where
+postings are stored.
+
+<b>The honest cost: total disk goes up.</b> The store is 3.89 MB against 1.31 MB of JSON postings,
+because SQLite keeps the term string on every row plus an index over it. Total on-disk is 4.65 MB
+versus 2.05 MB. This card says <b>&ldquo;target RSS, not bytes&rdquo;</b> and that is why the trade
+is the right way round &mdash; the hot path now reads kilobytes instead of materialising megabytes of
+Python objects &mdash; but it is a real cost, not a free win. Interning terms into a
+<code>term_id</code> table would recover most of it and is the obvious follow-up if the size matters.
+
+<b>Both halves are now required.</b> <code>ready()</code> checks for the postings store as well as
+the documents, so a docs-only index reports not-ready instead of silently scoring every query to
+zero hits. <code>INDEX_VERSION</code> 5 &rarr; 6 forces the one-time rebuild.
+
+<b>Still open:</b> <code>snip</code> text off the hot path &mdash; now only 14% of the JSON half, so
+much smaller than this card assumed &mdash; and the O(entire-corpus) incremental reindex. The latter
+is closer than it was: per-document term frequencies are now queryable by document rather than only
+recoverable by inverting every posting, so scoping the rebuild to the changed taps is a tractable
+follow-up rather than a format change.</p>
         <div class="meta">
           <span class="m">Complexity <b>L</b></span>
           <span class="m hi">Impact <b>High</b></span>