boost-skill-cli 1.0.319__tar.gz → 1.0.320__tar.gz

{boost_skill_cli-1.0.319/boost_skill_cli.egg-info → boost_skill_cli-1.0.320}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: boost-skill-cli
-Version: 1.0.319
+Version: 1.0.320
 Summary: boost — Homebrew for AI coding skills
 Author: Jonathan Reyes
 License: GNU GENERAL PUBLIC LICENSE

{boost_skill_cli-1.0.319 → boost_skill_cli-1.0.320}/boost_cli/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '1.0.319'
-__version_tuple__ = version_tuple = (1, 0, 319)
+__version__ = version = '1.0.320'
+__version_tuple__ = version_tuple = (1, 0, 320)
 
 __commit_id__ = commit_id = None

{boost_skill_cli-1.0.319 → boost_skill_cli-1.0.320}/boost_cli/core/rag.py

@@ -33,7 +33,7 @@ from . import ai, catalog, frontmatter, gitutil, paths, registry, util
 # v2: `snip` stores a larger head of the matched chunk (not just SNIP_WIDTH
 # chars) so `retrieve` can window it onto the query terms; bump forces a
 # one-time reindex.
-INDEX_VERSION = 3
+INDEX_VERSION = 4
 ENGINE = "bm25"
 
 # Chunking defaults (documented in docs/rag-architecture.md §4).
@@ -162,25 +162,49 @@ def index_path() -> Path:
     return paths.cache_dir() / "rag_index.json"
 
 
+def surface(entry: dict) -> str:
+    """The item's own identifying text, indexed alongside its body.
+
+    A chunked index got name matching for free: the name sat in whichever chunk
+    happened to contain it. One document per entry has to state it, and the
+    de-hyphenated form matters because `tokenize` does not split on hyphens —
+    without it the query "code reviewer" cannot match the item `code-reviewer`.
+    """
+    name = entry.get("name", "")
+    return " ".join([name, name.replace("-", " ").replace("_", " "),
+                     entry.get("description") or ""])
+
+
 def _make_docs(entries: List[dict], tap_paths: Dict[str, Path]) -> List[dict]:
-    """Chunk + tokenize every entry into scored documents (one per chunk)."""
+    """Tokenize every entry into ONE scored document.
+
+    Chunking cost far more than it bought. Each 1000-char window carried its own
+    copy of the shared per-doc fields and its own `snip`, which is how a 743-entry
+    corpus became 3,740 documents and how the roadmap item measured a real
+    83-tap install at 132 MB and 8-13 s of `json.loads` on *every cold search* —
+    against 31-70 ms of actual BM25 scoring.
+
+    The one thing chunking genuinely provided was locality: a term only had to
+    beat the length normalisation of its own window, not of an entire long
+    document. BM25's `b` parameter already handles that, and `retrieve` was
+    collapsing chunks back to one hit per entry anyway (max score across
+    chunks), so the extra documents were discarded at query time.
+    """
     docs: List[dict] = []
     for e in entries:
         body = read_body(e, tap_paths)
-        pieces = chunk(body) or [body]
-        for ci, piece in enumerate(pieces):
-            tf: Dict[str, int] = defaultdict(int)
-            for tok in tokenize(piece):
-                tf[tok] += 1
-            if not tf:
-                continue
-            docs.append({
-                "n": e["name"], "t": e["tap"], "f": e["skill_md"],
-                "k": e.get("kind", "skill"),
-                "c": ci, "l": sum(tf.values()),
-                "snip": piece[:SNIP_STORE].strip(),  # windowed at retrieve time
-                "tf": dict(tf),  # noqa: FURB123  tf is a defaultdict; .copy() would keep the factory
-            })
+        tf: Dict[str, int] = defaultdict(int)
+        for tok in tokenize(surface(e) + "\n" + body):
+            tf[tok] += 1
+        if not tf:
+            continue
+        docs.append({
+            "n": e["name"], "t": e["tap"], "f": e["skill_md"],
+            "k": e.get("kind", "skill"),
+            "c": 0, "l": sum(tf.values()),
+            "snip": body[:SNIP_STORE].strip(),  # windowed at retrieve time
+            "tf": dict(tf),  # noqa: FURB123  tf is a defaultdict; .copy() would keep the factory
+        })
     return docs
 
 

{boost_skill_cli-1.0.319 → boost_skill_cli-1.0.320/boost_skill_cli.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: boost-skill-cli
-Version: 1.0.319
+Version: 1.0.320
 Summary: boost — Homebrew for AI coding skills
 Author: Jonathan Reyes
 License: GNU GENERAL PUBLIC LICENSE

{boost_skill_cli-1.0.319 → boost_skill_cli-1.0.320}/docs/roadmap/items/bm25-index-is-one-json-blob.md

@@ -2,14 +2,14 @@
 id: bm25-index-is-one-json-blob
 board: code
 section: pipeline
-status: planned
+status: inflight
 category: Search · Index
 complexity: L
 impact: High
 wow: 5
 note: 2.5 GB RSS per search at 50k items
 order: 72
-owner:
+owner: loop/unchunked-bm25-index
 pr:
 title: The BM25 index is one JSON blob, and it stops working between 10k and 50k items
 ---
@@ -43,3 +43,33 @@ Also note <code>build()</code>'s incremental path is O(entire corpus): reusing
 unchanged taps runs <code>_kept_docs</code> → <code>_postings_to_doc_tf</code>,
 which inverts every posting in the index — measured 35 s and 1.72 GB RSS at
 11.1k entries to reindex a single changed file.
+
+<b>Win (1) is shipped, and the &ldquo;better on every metric&rdquo; claim did not survive
+re-measurement.</b> On the pinned 6-tap corpus unchunking is a <em>trade</em>, not a free win, and
+which way it falls depends entirely on the shape of the query.
+
+On the keyword golden set &mdash; which grades items by <b>name</b> &mdash; recall@10 fell
+1.000 &rarr; <b>0.978</b> and nDCG 0.895 &rarr; 0.882, while hit@1 rose 0.780 &rarr; <b>0.791</b>.
+On the 50 natural-language queries in <code>golden-natural.jsonl</code>, over identical data, every
+metric improved and not marginally: recall 0.690 &rarr; <b>0.750</b>, hit@1 0.240 &rarr; <b>0.340</b>,
+MRR 0.382 &rarr; 0.474, nDCG 0.446 &rarr; 0.524. That +0.100 hit@1 is 5 of 50 queries, which is at
+the edge of what 50 queries can resolve &mdash; worth stating rather than rounding up.
+
+Index cost on the same corpus: <b>5.3 MB &rarr; 2.1 MB</b> and 3,740 documents &rarr; 743, one per
+entry, with load time 0.032 s &rarr; 0.015 s. The card's headline figures are from an 83-tap install
+where the constant factors are far larger; this is the small-scale confirmation of the same shape,
+not a substitute for it.
+
+Chunking's one real contribution was locality &mdash; a term only had to beat the length
+normalisation of its own 1000-char window. BM25's <code>b</code> already does that, and
+<code>retrieve</code> collapsed chunks back to one hit per entry regardless, so the extra documents
+were built, stored, re-parsed on every cold search and then discarded. What chunking <em>did</em>
+provide for free was name matching, since the name sat in whichever chunk contained it; one document
+per entry has to state the surface explicitly, so <code>rag.surface</code> indexes the name, its
+de-hyphenated form (<code>tokenize</code> does not split hyphens, so &ldquo;code reviewer&rdquo;
+would otherwise never match <code>code-reviewer</code>) and the description alongside the body.
+
+<b>Still open:</b> win (2), getting postings out of JSON &mdash; the FTS5 probe and the mmap-able
+binary format are both untouched, as is moving <code>snip</code> text off the hot path. So is
+<code>build()</code>'s O(entire-corpus) incremental path, which still inverts every posting to
+reindex one changed file.

{boost_skill_cli-1.0.319 → boost_skill_cli-1.0.320}/docs/roadmap.html

@@ -3436,7 +3436,7 @@ that were never the same from being merged by accident.</p>
       </article>
 
       <article class="cap rcard" id="bm25-index-is-one-json-blob">
-        <div class="head"><span class="pill planned">Planned</span><span class="cat">Search · Index</span></div>
+        <div class="head"><span class="pill inflight">In flight</span><span class="cat">Search · Index</span></div>
         <h3>The BM25 index is one JSON blob, and it stops working between 10k and 50k items</h3>
         <p><code>rag.py</code> persists the whole index as a single JSON file and
 <code>json.loads</code> it on every invocation. <code>_CACHE</code>
@@ -3467,7 +3467,37 @@ out of the hot path.
 Also note <code>build()</code>'s incremental path is O(entire corpus): reusing
 unchanged taps runs <code>_kept_docs</code> → <code>_postings_to_doc_tf</code>,
 which inverts every posting in the index — measured 35 s and 1.72 GB RSS at
-11.1k entries to reindex a single changed file.</p>
+11.1k entries to reindex a single changed file.
+
+<b>Win (1) is shipped, and the &ldquo;better on every metric&rdquo; claim did not survive
+re-measurement.</b> On the pinned 6-tap corpus unchunking is a <em>trade</em>, not a free win, and
+which way it falls depends entirely on the shape of the query.
+
+On the keyword golden set &mdash; which grades items by <b>name</b> &mdash; recall@10 fell
+1.000 &rarr; <b>0.978</b> and nDCG 0.895 &rarr; 0.882, while hit@1 rose 0.780 &rarr; <b>0.791</b>.
+On the 50 natural-language queries in <code>golden-natural.jsonl</code>, over identical data, every
+metric improved and not marginally: recall 0.690 &rarr; <b>0.750</b>, hit@1 0.240 &rarr; <b>0.340</b>,
+MRR 0.382 &rarr; 0.474, nDCG 0.446 &rarr; 0.524. That +0.100 hit@1 is 5 of 50 queries, which is at
+the edge of what 50 queries can resolve &mdash; worth stating rather than rounding up.
+
+Index cost on the same corpus: <b>5.3 MB &rarr; 2.1 MB</b> and 3,740 documents &rarr; 743, one per
+entry, with load time 0.032 s &rarr; 0.015 s. The card's headline figures are from an 83-tap install
+where the constant factors are far larger; this is the small-scale confirmation of the same shape,
+not a substitute for it.
+
+Chunking's one real contribution was locality &mdash; a term only had to beat the length
+normalisation of its own 1000-char window. BM25's <code>b</code> already does that, and
+<code>retrieve</code> collapsed chunks back to one hit per entry regardless, so the extra documents
+were built, stored, re-parsed on every cold search and then discarded. What chunking <em>did</em>
+provide for free was name matching, since the name sat in whichever chunk contained it; one document
+per entry has to state the surface explicitly, so <code>rag.surface</code> indexes the name, its
+de-hyphenated form (<code>tokenize</code> does not split hyphens, so &ldquo;code reviewer&rdquo;
+would otherwise never match <code>code-reviewer</code>) and the description alongside the body.
+
+<b>Still open:</b> win (2), getting postings out of JSON &mdash; the FTS5 probe and the mmap-able
+binary format are both untouched, as is moving <code>snip</code> text off the hot path. So is
+<code>build()</code>'s O(entire-corpus) incremental path, which still inverts every posting to
+reindex one changed file.</p>
         <div class="meta">
           <span class="m">Complexity <b>L</b></span>
           <span class="m hi">Impact <b>High</b></span>

{boost_skill_cli-1.0.319 → boost_skill_cli-1.0.320}/tests/eval/baseline.json

@@ -11,10 +11,10 @@
           "nDCG@k": 0.8098096631332409
         },
         "BM25 full-content": {
-          "recall@k": 1.0,
-          "hit@1": 0.7802197802197802,
-          "MRR": 0.8604395604395604,
-          "nDCG@k": 0.8945515691995823
+          "recall@k": 0.978021978021978,
+          "hit@1": 0.7912087912087912,
+          "MRR": 0.8542124542124542,
+          "nDCG@k": 0.8823506423411086
         }
       }
     },
@@ -29,10 +29,10 @@
           "nDCG@k": 0.19698368469243419
         },
         "BM25 full-content": {
-          "recall@k": 0.69,
-          "hit@1": 0.24,
-          "MRR": 0.3824806870205885,
-          "nDCG@k": 0.44627899978118957
+          "recall@k": 0.75,
+          "hit@1": 0.34,
+          "MRR": 0.47449253613769743,
+          "nDCG@k": 0.5243426877966659
         }
       }
     }

{boost_skill_cli-1.0.319 → boost_skill_cli-1.0.320}/tests/unit/test_rag.py

@@ -605,18 +605,35 @@ class TestMakeDocs:
         docs = rag._make_docs([e], {})
         assert docs[0]["snip"] == "Q" * rag.SNIP_STORE
 
-    def test_long_body_yields_multiple_chunks(self, monkeypatch):
+    def test_one_document_per_entry_however_long_the_body(self, monkeypatch):
+        # Inverted deliberately. This asserted `len(docs) > 1`, the chunking the
+        # index no longer does: each window carried its own copy of the shared
+        # fields and its own snippet, and `retrieve` collapsed them back to one
+        # hit per entry anyway, so the extra documents were built, stored,
+        # re-parsed on every cold search, and then discarded.
         monkeypatch.setattr(rag, "read_body", lambda e, tp: "widget " * 400)
-        e = _entry("n")
-        docs = rag._make_docs([e], {})
-        assert len(docs) > 1                    # chunk() or [body], not `and`
+        docs = rag._make_docs([_entry("n")], {})
+        assert len(docs) == 1
+        assert docs[0]["tf"]["widget"] == 400
+
+    def test_the_entry_surface_is_indexed_with_the_body(self, monkeypatch):
+        # A chunked index matched names for free — the name sat in whatever
+        # chunk contained it. One doc per entry has to state the surface, or
+        # searching for an item by its own name stops working.
+        monkeypatch.setattr(rag, "read_body", lambda e, tp: "unrelated prose")
+        docs = rag._make_docs(
+            [_entry("code-reviewer", desc="reviews diffs")], {})
+        tf = docs[0]["tf"]
+        assert "reviewer" in tf, "the de-hyphenated name must be searchable"
+        assert "diffs" in tf, "the description must be searchable"
 
     def test_empty_chunk_is_skipped_not_break(self, monkeypatch):
         body = ("the " * 220) + "\n\n" + ("widget " * 130)
         monkeypatch.setattr(rag, "read_body", lambda e, tp: body)
         e = _entry("n")
         docs = rag._make_docs([e], {})
-        # first chunk is all stopwords (empty tf) -> `continue`, keep going
+        # An all-stopword passage contributes no terms; the entry still yields
+        # its one document, carrying the terms from the rest of the body.
         assert len(docs) == 1
         assert "widget" in docs[0]["tf"]