boost-skill-cli 1.2.106__tar.gz → 1.2.108__tar.gz

{boost_skill_cli-1.2.106/boost_skill_cli.egg-info → boost_skill_cli-1.2.108}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: boost-skill-cli
-Version: 1.2.106
+Version: 1.2.108
 Summary: boost — Homebrew for AI coding skills
 Author: Jonathan Reyes
 License-Expression: Apache-2.0

{boost_skill_cli-1.2.106 → boost_skill_cli-1.2.108}/boost_cli/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '1.2.106'
-__version_tuple__ = version_tuple = (1, 2, 106)
+__version__ = version = '1.2.108'
+__version_tuple__ = version_tuple = (1, 2, 108)
 
 __commit_id__ = commit_id = None

{boost_skill_cli-1.2.106 → boost_skill_cli-1.2.108}/boost_cli/commands/discovery.py

@@ -573,6 +573,20 @@ def cmd_reindex(argv):
                        % (len(stats["reused"]),
                           "" if len(stats["reused"]) == 1 else "s",
                           ", ".join(stats["reindexed"]) or "none"), "muted"))
+    # The line that was missing. `boost catalog --import` restores catalogues
+    # with no repositories cloned, so every entry indexes from its frontmatter
+    # and the count above reads exactly as it does for a full corpus. Say what
+    # the index actually holds, in the share of TEXT rather than of documents —
+    # a document share reads 100% until it reads 0%.
+    if stats["metadata_only"]:
+        share = rag.index_completeness() or {}
+        out.warn("%d of %d items were indexed from catalog metadata alone — "
+                 "their registries are not cloned here, so this index holds "
+                 "%.1f%% of the searchable text. Tap them for real with "
+                 "`boost tap <owner/repo>` and re-run `boost reindex` to index "
+                 "their bodies."
+                 % (stats["metadata_only"], stats["docs"],
+                    100.0 * share.get("body_share", 0.0)), wrap=True)
     if args.dense:
         if dense_stats is None:
             out.warn("dense index skipped — %s" % embed.fallback_note(),

{boost_skill_cli-1.2.106 → boost_skill_cli-1.2.108}/boost_cli/core/rag.py

@@ -51,7 +51,7 @@ from . import ai, catalog, config, frontmatter, gitutil, paths, registry, util
 # reach the entries spelling it that way. `build` reuses a tap only when
 # `_load_raw` returned an index, and a version mismatch returns None — so the
 # bump re-indexes every tap rather than half of them.
-INDEX_VERSION = 8
+INDEX_VERSION = 9
 ENGINE = "bm25"
 
 # Chunking defaults (documented in docs/rag-architecture.md §4).
@@ -294,23 +294,51 @@ def entry_path(entry: dict, tap_paths: dict[str, Path] | None = None
     return Path(base) / rel
 
 
-def read_body(entry: dict, tap_paths: dict[str, Path] | None = None) -> str:
-    """Return an item's searchable text: name + description + prose body.
-
-    The frontmatter is stripped (reusing ``frontmatter.parse``); the name and
-    description are prepended so short items keep keyword parity with the old
-    search. Missing files degrade to just the catalog metadata.
+def read_body_full(entry: dict, tap_paths: dict[str, Path] | None = None
+                   ) -> tuple[str, bool]:
+    """Return ``(text, has_body)`` — the searchable text, and whether it is
+    actually the item's body rather than its catalog metadata standing in.
+
+    The degradation itself is unchanged and deliberate: a tap whose clone is
+    absent still contributes its name and description, because a smaller index
+    beats no index. What was missing is that it happened *silently*.
+    ``boost catalog --import`` restores catalogues with zero repositories
+    cloned, so every entry takes this path, and the index it produces is not
+    the full-content index the ``evals`` gate floors — it is a frontmatter
+    index wearing the same file name. Measured over 3,015 real entries,
+    indexed with and then without their clones: 3,041,326 tokens against
+    182,507, or **6.0%** of the searchable text, with nothing in the output
+    saying so.
+
+    ``has_body`` is False in both degrade cases, and they are one answer on
+    purpose: whether the entry names no file or names one that cannot be read,
+    the document carries only metadata, which is the thing a caller needs to
+    know. :func:`build` counts them and :func:`index_completeness` reports the
+    share.
     """
     header = "%s\n%s" % (entry.get("name", ""), entry.get("description", ""))
     src = entry_path(entry, tap_paths)
     if src is None:
-        return header.strip()
+        return header.strip(), False
     try:
         text = src.read_text(encoding="utf-8", errors="replace")
     except OSError:
-        return header.strip()
+        return header.strip(), False
     _meta, body = frontmatter.parse(text)
-    return ("%s\n%s" % (header, body)).strip()
+    return ("%s\n%s" % (header, body)).strip(), True
+
+
+def read_body(entry: dict, tap_paths: dict[str, Path] | None = None) -> str:
+    """An item's searchable text: name + description + prose body.
+
+    The frontmatter is stripped (reusing ``frontmatter.parse``); the name and
+    description are prepended so short items keep keyword parity with the old
+    search. Missing files degrade to just the catalog metadata — ask
+    :func:`read_body_full` when you need to know that it happened. This
+    signature is load-bearing: ``dense`` and ``boost_langchain`` both call it
+    and several tests monkeypatch it.
+    """
+    return read_body_full(entry, tap_paths)[0]
 
 
 # ----------------------------------------------------------------- build
@@ -349,7 +377,7 @@ def _make_docs(entries: list[dict], tap_paths: dict[str, Path]) -> list[dict]:
     """
     docs: list[dict] = []
     for e in entries:
-        body = read_body(e, tap_paths)
+        body, has_body = read_body_full(e, tap_paths)
         # The scanner already hashed this exact text (`catalog._content_digest`
         # assembles name + description + body, the same string `read_body`
         # returns), so an entry from a current cache hands the value over. The
@@ -365,6 +393,13 @@ def _make_docs(entries: list[dict], tap_paths: dict[str, Path]) -> list[dict]:
         docs.append({
             "n": e["name"], "t": e["tap"], "f": e["skill_md"], "h": digest,
             "k": e.get("kind", "skill"),
+            # Written only when the body is MISSING, so a fully cloned corpus
+            # pays nothing for the flag — which matters on an index measured at
+            # 43.7 MB. Absence therefore means "has a body", and that reading
+            # is only safe because INDEX_VERSION moved with it: `_load_raw`
+            # rejects any other version outright, so no document written before
+            # this change can be misread as complete.
+            **({"m": 1} if not has_body else {}),
             "c": 0, "l": sum(tf.values()),
             "snip": body[:SNIP_STORE].strip(),  # windowed at retrieve time
             "tf": dict(tf),  # noqa: FURB123  tf is a defaultdict; .copy() would keep the factory
@@ -428,7 +463,7 @@ def build(entries: list[dict] | None = None, force: bool = False) -> dict:
     if old is not None and reused_safe:
         docs = _kept_docs(old, reused_safe) + docs
 
-    _save(docs, commits)
+    saved = _save(docs, commits)
     reindexed = sorted({e["tap"] for e in fresh})
     # `reindexed` names taps by their real name ("owner/repo"); `reused_safe`
     # is keyed by the safe name ("owner__repo") that `_tap_commits` uses to
@@ -446,6 +481,10 @@ def build(entries: list[dict] | None = None, force: bool = False) -> dict:
         "taps": len(commits),
         "reindexed": reindexed,
         "reused": reused,
+        # Counted over every document written, reused ones included — an
+        # incremental build that reported only what it re-indexed would say
+        # zero on the run after a bundle import.
+        "metadata_only": saved["metadata_only"],
     }
 
 
@@ -640,17 +679,38 @@ def _all_postings() -> dict[str, list[list[int]]]:
     return dict(out)  # noqa: FURB123  out is a defaultdict; .copy() would keep the factory
 
 
-def _save(docs: list[dict], commits: dict[str, str]) -> None:
+def _save(docs: list[dict], commits: dict[str, str]) -> dict:
+    """Persist the index and return the ``stats`` block it wrote.
+
+    Returning the stats rather than recomputing them in :func:`build` keeps one
+    definition of "how much of this index is real body text": the totals are
+    summed here, over the same documents that are being written.
+    """
     postings: dict[str, list[list[int]]] = defaultdict(list)
     meta_docs: list[dict] = []
     total_len = 0
+    metadata_only = 0
+    metadata_only_len = 0
     for doc_id, d in enumerate(docs):
         for term, tf in d["tf"].items():
             postings[term].append([doc_id, tf])
         total_len += d["l"]
+        # `.get`, because a document reused from the previous index carries
+        # whatever it was written with, and a hand-built one (several tests)
+        # carries nothing. Absence is "has a body" — see `_make_docs`.
+        bodyless = bool(d.get("m"))
+        if bodyless:
+            metadata_only += 1
+            metadata_only_len += d["l"]
         meta_docs.append({"n": d["n"], "t": d["t"], "f": d["f"], "k": d["k"],
                           "h": d.get("h", ""),
+                          **({"m": 1} if bodyless else {}),
                           "c": d["c"], "l": d["l"], "snip": d["snip"]})
+    stats: dict = {"docs": len(docs),
+                   "avg_len": (total_len / len(docs)) if docs else 0.0,
+                   "tokens": total_len,
+                   "metadata_only": metadata_only,
+                   "metadata_only_tokens": metadata_only_len}
     payload = {
         "version": INDEX_VERSION,
         "engine": ENGINE,
@@ -658,8 +718,7 @@ def _save(docs: list[dict], commits: dict[str, str]) -> None:
         "commits": commits,
         "params": {"chunk_chars": CHUNK_CHARS, "overlap": OVERLAP,
                    "k1": K1, "b": B},
-        "stats": {"docs": len(docs),
-                  "avg_len": (total_len / len(docs)) if docs else 0.0},
+        "stats": stats,
         "docs": meta_docs,
     }
     paths.ensure_dirs()
@@ -671,6 +730,7 @@ def _save(docs: list[dict], commits: dict[str, str]) -> None:
     # Drop the mtime-keyed cache so a reindex is visible to an immediately
     # following query even when the filesystem mtime granularity is coarse.
     _CACHE.pop(str(p), None)
+    return stats
 
 
 def _now() -> str:
@@ -718,6 +778,43 @@ def ready() -> bool:
     return bool(raw and raw.get("docs") and postings_path().exists())
 
 
+def index_completeness() -> dict | None:
+    """How much of the index on disk is the corpus, and how much is its labels.
+
+    Returns ``None`` when there is no readable index — the same answer
+    :func:`_load_raw` gives, rather than a zeroed dict, because "no index" and
+    "an index carrying no bodies" are different situations with different
+    remedies and must not render alike.
+
+    ``body_share`` is a share of **tokens**, not of documents, and that is the
+    whole point. An entry whose body is missing still produces a document, so a
+    document share sits at 1.0 until the moment it drops to 0.0 and tells a
+    partially-cloned machine nothing. The token share degrades smoothly and
+    reproduces the measurement the roadmap card made by hand: 3,041,326 tokens
+    against 182,507 over 3,015 entries indexed with and then without their
+    clones, or 6.0% of the searchable text.
+
+    Reads the persisted totals rather than re-deriving them, so asking is a
+    stat plus a cached parse — nothing walks the corpus.
+    """
+    raw = _load_raw()
+    if raw is None:
+        return None
+    stats = raw.get("stats") or {}
+    tokens = int(stats.get("tokens") or 0)
+    metadata_only_tokens = int(stats.get("metadata_only_tokens") or 0)
+    return {
+        "docs": int(stats.get("docs") or 0),
+        "metadata_only": int(stats.get("metadata_only") or 0),
+        "tokens": tokens,
+        "metadata_only_tokens": metadata_only_tokens,
+        # Guarded on `tokens`, not on `docs`: an index of documents that all
+        # tokenized to nothing would divide by zero, and 0.0 is the honest
+        # answer there — no body text is present.
+        "body_share": (1.0 - metadata_only_tokens / tokens) if tokens else 0.0,
+    }
+
+
 def stale() -> bool:
     """True when the index no longer reflects the taps on disk.
 

{boost_skill_cli-1.2.106 → boost_skill_cli-1.2.108/boost_skill_cli.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: boost-skill-cli
-Version: 1.2.106
+Version: 1.2.108
 Summary: boost — Homebrew for AI coding skills
 Author: Jonathan Reyes
 License-Expression: Apache-2.0

{boost_skill_cli-1.2.106 → boost_skill_cli-1.2.108}/boost_skill_cli.egg-info/SOURCES.txt

@@ -956,6 +956,7 @@ tests/unit/test_projectlock.py
 tests/unit/test_provenance.py
 tests/unit/test_publish_golden_dataset.py
 tests/unit/test_rag.py
+tests/unit/test_rag_completeness.py
 tests/unit/test_rag_fastpath.py
 tests/unit/test_rag_fusion.py
 tests/unit/test_registries_fresh.py

{boost_skill_cli-1.2.106 → boost_skill_cli-1.2.108}/docs/roadmap/items/publish-the-keyword-index.md

@@ -2,13 +2,14 @@
 id: publish-the-keyword-index
 board: code
 section: planned
-status: planned
+status: inflight
 category: Search · Performance
 complexity: L
 impact: High
 wow: 4
 note: every machine rebuilds 697 MB of index for a corpus that is byte-identical on all of them
 order: 99
+owner: loop/keyword-index-completeness
 title: publish the keyword index the way vectors are published
 ---
 Dense vectors are built once in CI and downloaded. The BM25 index is not:
@@ -66,3 +67,27 @@ as a match.
 <a href="#shrink-the-published-index">shrink-the-published-index</a>. This card should not ship
 until that one has an answer, because publishing 697&nbsp;MB per refresh to save 65&nbsp;s of CPU
 is not obviously the right trade, and at the compressed sizes measured there it clearly is.
+
+<b>Partly landed, and deliberately still <code>inflight</code> — 2026-09-10.</b> What shipped is
+the half that needed no size decision: <b>the index now records what it is</b>.
+<code>read_body_full</code> returns the text and whether it contains the item's body,
+<code>build()</code> reports <code>metadata_only</code> over every document written (reused ones
+included, or an incremental build reports zero on the run after a bundle import),
+<code>index_completeness()</code> reads the share back off disk, and <code>boost reindex</code>
+says it out loud instead of reporting the same confident count for a 6% index. The share is of
+<b>tokens</b>, not documents: a bodyless entry still produces a document, so a document share sits
+at 1.0 until it drops to 0.0. <code>INDEX_VERSION</code> moved to <b>9</b>, because the flag is
+written only when a body is missing and absence may only be read as "complete" once no older
+document can survive.
+
+<b>What did NOT land:</b> <code>rag.export_shard</code> / <code>rag.import_shard</code>, the
+per-registry assets, and the <code>manifest.json</code> rows — the publishing pipeline itself.
+That half is what the payload-size question governs, and
+<a href="#shrink-the-published-index">shrink-the-published-index</a> still has no answer: its claim
+is <b>stale</b>, not active — branch <code>loop/shrink-postings-index</code> was last touched
+2026-09-02, carries one commit, has no pull request, and is 394 commits behind
+<code>main</code>. Someone should un-claim it. One structural finding for whoever takes it: doc
+ids are positional (<code>_save</code> does <code>enumerate(docs)</code>), so the card's
+"merge by offsetting <code>doc_id</code>" is sound as written — and the shard format should
+serialize <em>logical</em> postings (digest → term → tf) rather than the SQLite layout, so the
+interning that branch was attempting cannot invalidate a published shard.

{boost_skill_cli-1.2.106 → boost_skill_cli-1.2.108}/docs/roadmap.html

@@ -726,7 +726,7 @@ published version works against PyPI as it stands today.</p>
       </article>
 
       <article class="cap rcard" id="publish-the-keyword-index">
-        <div class="head"><span class="pill planned">Planned</span><span class="cat">Search · Performance</span></div>
+        <div class="head"><span class="pill inflight">In flight</span><span class="cat">Search · Performance</span></div>
         <h3>publish the keyword index the way vectors are published</h3>
         <p>Dense vectors are built once in CI and downloaded. The BM25 index is not:
 <code>core/rag.py</code> has <b>no export or import function at all</b>, and
@@ -782,7 +782,31 @@ as a match.
 <b>The open question is payload size</b>, and it is large enough to be its own decision — see
 <a href="#shrink-the-published-index">shrink-the-published-index</a>. This card should not ship
 until that one has an answer, because publishing 697&nbsp;MB per refresh to save 65&nbsp;s of CPU
-is not obviously the right trade, and at the compressed sizes measured there it clearly is.</p>
+is not obviously the right trade, and at the compressed sizes measured there it clearly is.
+
+<b>Partly landed, and deliberately still <code>inflight</code> — 2026-09-10.</b> What shipped is
+the half that needed no size decision: <b>the index now records what it is</b>.
+<code>read_body_full</code> returns the text and whether it contains the item's body,
+<code>build()</code> reports <code>metadata_only</code> over every document written (reused ones
+included, or an incremental build reports zero on the run after a bundle import),
+<code>index_completeness()</code> reads the share back off disk, and <code>boost reindex</code>
+says it out loud instead of reporting the same confident count for a 6% index. The share is of
+<b>tokens</b>, not documents: a bodyless entry still produces a document, so a document share sits
+at 1.0 until it drops to 0.0. <code>INDEX_VERSION</code> moved to <b>9</b>, because the flag is
+written only when a body is missing and absence may only be read as "complete" once no older
+document can survive.
+
+<b>What did NOT land:</b> <code>rag.export_shard</code> / <code>rag.import_shard</code>, the
+per-registry assets, and the <code>manifest.json</code> rows — the publishing pipeline itself.
+That half is what the payload-size question governs, and
+<a href="#shrink-the-published-index">shrink-the-published-index</a> still has no answer: its claim
+is <b>stale</b>, not active — branch <code>loop/shrink-postings-index</code> was last touched
+2026-09-02, carries one commit, has no pull request, and is 394 commits behind
+<code>main</code>. Someone should un-claim it. One structural finding for whoever takes it: doc
+ids are positional (<code>_save</code> does <code>enumerate(docs)</code>), so the card's
+"merge by offsetting <code>doc_id</code>" is sound as written — and the shard format should
+serialize <em>logical</em> postings (digest → term → tf) rather than the SQLite layout, so the
+interning that branch was attempting cannot invalidate a published shard.</p>
         <div class="meta">
           <span class="m">Complexity <b>L</b></span>
           <span class="m hi">Impact <b>High</b></span>

{boost_skill_cli-1.2.106 → boost_skill_cli-1.2.108}/tests/functional/test_cli_discovery.py

@@ -2088,6 +2088,35 @@ class TestReindex:
         data = json.loads(boost("reindex", "--force", "--json").out)
         assert data["reused"] == []
 
+    def test_an_uncloned_registry_is_named_rather_than_counted_as_full(
+            self, boost, tapped):
+        """The `boost catalog --import` shape: catalogs present, clones gone.
+
+        `rag.read_body` degrades to name + description without saying so, so
+        this index used to report the same confident "indexed N passages" as a
+        fully cloned one while holding a fraction of the text.
+        """
+        import shutil
+
+        from boost_cli.core import paths
+        boost("reindex")
+        for d in paths.repos_dir().iterdir():
+            shutil.rmtree(d)
+        data = json.loads(boost("reindex", "--force", "--json").out)
+        assert data["docs"] > 0
+        assert data["metadata_only"] == data["docs"]
+        r = boost("reindex", "--force")
+        said = r.out + r.err
+        assert "indexed from catalog metadata alone" in said
+        assert "0.0% of the searchable text" in said
+        assert "boost tap <owner/repo>" in said
+
+    def test_a_cloned_registry_says_nothing_about_metadata(self, boost, tapped):
+        r = boost("reindex", "--force")
+        assert "catalog metadata alone" not in (r.out + r.err)
+        assert json.loads(
+            boost("reindex", "--force", "--json").out)["metadata_only"] == 0
+
     def test_no_taps_errors(self, boost):
         r = boost("reindex", expect=1)
         assert "no taps configured" in r.err

{boost_skill_cli-1.2.106 → boost_skill_cli-1.2.108}/tests/unit/test_rag.py

@@ -1096,8 +1096,8 @@ class TestMakeDocs:
         assert "blueberry" in terms
 
     def test_term_frequencies_and_kind_default_and_snippet(self, monkeypatch):
-        monkeypatch.setattr(rag, "read_body",
-                            lambda e, tp: "widget widget gadget")
+        monkeypatch.setattr(rag, "read_body_full",
+                            lambda e, tp: ("widget widget gadget", True))
         e = {"name": "n", "tap": "x/y", "skill_md": "s"}  # no kind
         docs = rag._make_docs([e], {})
         assert docs[0]["tf"] == {"widget": 2, "gadget": 1}
@@ -1107,14 +1107,14 @@ class TestMakeDocs:
         # v2: more than SNIP_WIDTH is stored (windowed later at retrieve time)
         # so `retrieve` can center on the matched terms — but capped at
         # SNIP_STORE to bound index growth.
-        monkeypatch.setattr(rag, "read_body", lambda e, tp: "Q" * 250)
+        monkeypatch.setattr(rag, "read_body_full", lambda e, tp: ("Q" * 250, True))
         e = _entry("n")
         docs = rag._make_docs([e], {})
         assert docs[0]["snip"] == "Q" * 250              # under cap -> stored whole
 
     def test_snippet_storage_is_capped(self, monkeypatch):
-        monkeypatch.setattr(rag, "read_body",
-                            lambda e, tp: "Q" * (rag.SNIP_STORE + 200))
+        monkeypatch.setattr(rag, "read_body_full",
+                            lambda e, tp: ("Q" * (rag.SNIP_STORE + 200), True))
         e = _entry("n")
         docs = rag._make_docs([e], {})
         assert docs[0]["snip"] == "Q" * rag.SNIP_STORE
@@ -1125,7 +1125,7 @@ class TestMakeDocs:
         # fields and its own snippet, and `retrieve` collapsed them back to one
         # hit per entry anyway, so the extra documents were built, stored,
         # re-parsed on every cold search, and then discarded.
-        monkeypatch.setattr(rag, "read_body", lambda e, tp: "widget " * 400)
+        monkeypatch.setattr(rag, "read_body_full", lambda e, tp: ("widget " * 400, True))
         docs = rag._make_docs([_entry("n")], {})
         assert len(docs) == 1
         assert docs[0]["tf"]["widget"] == 400
@@ -1134,7 +1134,7 @@ class TestMakeDocs:
         # A chunked index matched names for free — the name sat in whatever
         # chunk contained it. One doc per entry has to state the surface, or
         # searching for an item by its own name stops working.
-        monkeypatch.setattr(rag, "read_body", lambda e, tp: "unrelated prose")
+        monkeypatch.setattr(rag, "read_body_full", lambda e, tp: ("unrelated prose", True))
         docs = rag._make_docs(
             [_entry("code-reviewer", desc="reviews diffs")], {})
         tf = docs[0]["tf"]
@@ -1143,7 +1143,7 @@ class TestMakeDocs:
 
     def test_empty_chunk_is_skipped_not_break(self, monkeypatch):
         body = ("the " * 220) + "\n\n" + ("widget " * 130)
-        monkeypatch.setattr(rag, "read_body", lambda e, tp: body)
+        monkeypatch.setattr(rag, "read_body_full", lambda e, tp: (body, True))
         e = _entry("n")
         docs = rag._make_docs([e], {})
         # An all-stopword passage contributes no terms; the entry still yields