boost-skill-cli 1.2.82__tar.gz → 1.2.83__tar.gz

{boost_skill_cli-1.2.82/boost_skill_cli.egg-info → boost_skill_cli-1.2.83}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: boost-skill-cli
-Version: 1.2.82
+Version: 1.2.83
 Summary: boost — Homebrew for AI coding skills
 Author: Jonathan Reyes
 License-Expression: Apache-2.0

{boost_skill_cli-1.2.82 → boost_skill_cli-1.2.83}/boost_cli/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '1.2.82'
-__version_tuple__ = version_tuple = (1, 2, 82)
+__version__ = version = '1.2.83'
+__version_tuple__ = version_tuple = (1, 2, 83)
 
 __commit_id__ = commit_id = None

{boost_skill_cli-1.2.82 → boost_skill_cli-1.2.83}/boost_cli/core/rag.py

@@ -41,7 +41,10 @@ from . import ai, catalog, config, frontmatter, gitutil, paths, registry, util
 # v2: `snip` stores a larger head of the matched chunk (not just SNIP_WIDTH
 # chars) so `retrieve` can window it onto the query terms; bump forces a
 # one-time reindex.
-INDEX_VERSION = 6
+# v7: postings interns term text into its own `terms` table instead of
+# repeating it on every posting row — see `_write_postings`; bump forces a
+# one-time reindex so every store gets the smaller layout.
+INDEX_VERSION = 7
 ENGINE = "bm25"
 
 # Chunking defaults (documented in docs/rag-architecture.md §4).
@@ -331,6 +334,14 @@ def _write_postings(postings: dict[str, list[list[int]]]) -> None:
     SQLite is used purely as an indexed key-value store here, not as a search
     engine: BM25 scoring stays in `_bm25` byte for byte, so this changes cold
     start cost and nothing about which results come back.
+
+    Terms are interned into their own table rather than repeated on every
+    posting row. Measured on a real 458-tap store: 18,619,658 posting rows over
+    210,422 distinct terms averaging 6.6 characters — the term string alone was
+    ~123 MB of the 653 MB file, stored on average 88 times over. `postings` now
+    carries an integer `term_id`, and `terms` also carries each term's document
+    frequency (`df`, the length of its posting list) so `stem_expansions` can
+    read it directly instead of re-deriving it with `GROUP BY` on every lookup.
     """
     paths.ensure_dirs()
     final = postings_path()
@@ -343,13 +354,22 @@ def _write_postings(postings: dict[str, list[list[int]]]) -> None:
         # mid-build leaves the .tmp behind rather than a torn index.
         con.execute("PRAGMA journal_mode=OFF")
         con.execute("PRAGMA synchronous=OFF")
-        con.execute("CREATE TABLE postings (term TEXT, doc INTEGER, tf INTEGER)")
+        con.execute("CREATE TABLE terms (id INTEGER PRIMARY KEY, "
+                    "term TEXT NOT NULL, df INTEGER NOT NULL)")
+        con.execute("CREATE TABLE postings (term_id INTEGER, doc INTEGER, "
+                    "tf INTEGER)")
+        items = list(postings.items())
+        con.executemany(
+            "INSERT INTO terms (id, term, df) VALUES (?, ?, ?)",
+            ((term_id, term, len(plist))
+             for term_id, (term, plist) in enumerate(items)))
         con.executemany(
-            "INSERT INTO postings (term, doc, tf) VALUES (?, ?, ?)",
-            ((term, doc, tf) for term, plist in postings.items()
+            "INSERT INTO postings (term_id, doc, tf) VALUES (?, ?, ?)",
+            ((term_id, doc, tf) for term_id, (_term, plist) in enumerate(items)
              for doc, tf in plist))
         # Built after the insert: maintaining it per-row is far slower.
-        con.execute("CREATE INDEX postings_term ON postings(term)")
+        con.execute("CREATE UNIQUE INDEX terms_term ON terms(term)")
+        con.execute("CREATE INDEX postings_term_id ON postings(term_id)")
         con.commit()
     finally:
         con.close()
@@ -375,8 +395,9 @@ def read_postings(terms: Sequence[str]) -> dict[str, list[list[int]]]:
         # long query can exceed it.
         for i in range(0, len(uniq), 500):
             batch = uniq[i:i + 500]
-            q = ("SELECT term, doc, tf FROM postings WHERE term IN (%s)"  # noqa: S608  interpolates only `?` placeholders; terms are bound params
-                 % ",".join("?" * len(batch)))
+            q = ("SELECT t.term, p.doc, p.tf FROM postings p "  # noqa: S608  interpolates only `?` placeholders; terms are bound params
+                 "JOIN terms t ON p.term_id = t.id "
+                 "WHERE t.term IN (%s)" % ",".join("?" * len(batch)))
             for term, doc, tf in con.execute(q, batch):
                 out[term].append([doc, tf])
     except sqlite3.Error:
@@ -429,12 +450,14 @@ def stem_expansions(terms: Sequence[str]) -> dict[str, str]:
         for term in missing:
             # `tokenize` emits only [a-z0-9]+, and "~" sorts above every
             # character it can produce — so this half-open range is exactly the
-            # terms beginning with `term`, and it rides the postings_term index
-            # instead of scanning. Ties break on the term itself so the choice
-            # is deterministic across machines.
+            # terms beginning with `term`, and it rides the terms_term index
+            # instead of scanning. `df` (document frequency) is stored per term
+            # at build time, so this needs no join and no GROUP BY. Ties break
+            # on the term itself so the choice is deterministic across
+            # machines.
             row = con.execute(
-                "SELECT term FROM postings WHERE term > ? AND term < ? "
-                "GROUP BY term ORDER BY COUNT(*) DESC, term LIMIT 1",
+                "SELECT term FROM terms WHERE term > ? AND term < ? "
+                "ORDER BY df DESC, term LIMIT 1",
                 (term, term + "~")).fetchone()
             if row:
                 out[term] = row[0]
@@ -453,7 +476,9 @@ def _all_postings() -> dict[str, list[list[int]]]:
         return {}
     out: dict[str, list[list[int]]] = defaultdict(list)
     try:
-        for term, doc, tf in con.execute("SELECT term, doc, tf FROM postings"):
+        for term, doc, tf in con.execute(
+                "SELECT t.term, p.doc, p.tf FROM postings p "
+                "JOIN terms t ON p.term_id = t.id"):
             out[term].append([doc, tf])
     except sqlite3.Error:
         return {}

{boost_skill_cli-1.2.82 → boost_skill_cli-1.2.83/boost_skill_cli.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: boost-skill-cli
-Version: 1.2.82
+Version: 1.2.83
 Summary: boost — Homebrew for AI coding skills
 Author: Jonathan Reyes
 License-Expression: Apache-2.0

{boost_skill_cli-1.2.82 → boost_skill_cli-1.2.83}/docs/roadmap/items/audit-tap-dry-run-is-silently-ignored-outside-catalog-spec-and-def.md

@@ -2,7 +2,7 @@
 id: audit-tap-dry-run-is-silently-ignored-outside-catalog-spec-and-def
 board: code
 section: dx
-status: inflight
+status: shipped
 category: Safety · Bug
 complexity: S
 impact: High

{boost_skill_cli-1.2.82 → boost_skill_cli-1.2.83}/docs/roadmap/items/shrink-the-published-index.md

@@ -2,13 +2,14 @@
 id: shrink-the-published-index
 board: code
 section: planned
-status: planned
+status: inflight
 category: Performance · Storage
 complexity: M
 impact: Med
 wow: 3
 note: 653 MB of postings holds 1.4 MB of distinct terms — the term string is stored 88 times over
 order: 100
+owner: loop/shrink-postings-index
 title: shrink the keyword index before publishing it — structure first, then compression
 ---
 <a href="#publish-the-keyword-index">publish-the-keyword-index</a> is worth doing only if the
@@ -58,3 +59,17 @@ for the many-small-registries case, where per-shard compression has little conte
 — on the real store, with import-side decode time beside each. That table is what tells
 <a href="#publish-the-keyword-index">publish-the-keyword-index</a> what to ship, and it is worth
 having even if publishing is declined: the on-disk win applies to every install today.
+
+<b>Progress (PR pending).</b> The structural half shipped: <code>_write_postings</code> now
+interns terms into their own <code>terms(id, term, df)</code> table, with <code>postings</code>
+carrying an integer <code>term_id</code> instead of repeating the term string on every row —
+exactly the "structure first" change this card calls the bigger win, and it bumps
+<code>INDEX_VERSION</code> so every store picks it up on its next rebuild.
+<code>stem_expansions</code> now reads the precomputed <code>df</code> column directly instead of
+a <code>GROUP BY COUNT(*)</code> over <code>postings</code> on every prefix lookup. Not done: the
+delta/varint doc-id encoding, and the full raw/interned/delta+varint &times;
+none/gzip/zstd comparison table with import-side decode times on a real multi-hundred-MB store —
+this sandbox has no such store to measure against, only a small synthetic one (interning alone cut
+a 1.8M-posting/20k-term synthetic store from 63.6&nbsp;MB to 47.9&nbsp;MB, directionally consistent
+with the real-store estimate above but not a substitute for it). Left as follow-on work before this
+card can be called shipped.

{boost_skill_cli-1.2.82 → boost_skill_cli-1.2.83}/docs/roadmap.html

@@ -1078,7 +1078,7 @@ is not obviously the right trade, and at the compressed sizes measured there it
       </article>
 
       <article class="cap rcard" id="shrink-the-published-index">
-        <div class="head"><span class="pill planned">Planned</span><span class="cat">Performance · Storage</span></div>
+        <div class="head"><span class="pill inflight">In flight</span><span class="cat">Performance · Storage</span></div>
         <h3>shrink the keyword index before publishing it — structure first, then compression</h3>
         <p><a href="#publish-the-keyword-index">publish-the-keyword-index</a> is worth doing only if the
 artifact is small enough to ship weekly. This card is the measurement that decides it, and the
@@ -1126,7 +1126,21 @@ for the many-small-registries case, where per-shard compression has little conte
 <b>Deliverable.</b> A measured comparison — raw, interned, delta+varint, each &times; none/gzip/zstd
 — on the real store, with import-side decode time beside each. That table is what tells
 <a href="#publish-the-keyword-index">publish-the-keyword-index</a> what to ship, and it is worth
-having even if publishing is declined: the on-disk win applies to every install today.</p>
+having even if publishing is declined: the on-disk win applies to every install today.
+
+<b>Progress (PR pending).</b> The structural half shipped: <code>_write_postings</code> now
+interns terms into their own <code>terms(id, term, df)</code> table, with <code>postings</code>
+carrying an integer <code>term_id</code> instead of repeating the term string on every row —
+exactly the "structure first" change this card calls the bigger win, and it bumps
+<code>INDEX_VERSION</code> so every store picks it up on its next rebuild.
+<code>stem_expansions</code> now reads the precomputed <code>df</code> column directly instead of
+a <code>GROUP BY COUNT(*)</code> over <code>postings</code> on every prefix lookup. Not done: the
+delta/varint doc-id encoding, and the full raw/interned/delta+varint &times;
+none/gzip/zstd comparison table with import-side decode times on a real multi-hundred-MB store —
+this sandbox has no such store to measure against, only a small synthetic one (interning alone cut
+a 1.8M-posting/20k-term synthetic store from 63.6&nbsp;MB to 47.9&nbsp;MB, directionally consistent
+with the real-store estimate above but not a substitute for it). Left as follow-on work before this
+card can be called shipped.</p>
         <div class="meta">
           <span class="m">Complexity <b>M</b></span>
           <span class="m">Impact <b>Med</b></span>
@@ -11982,15 +11996,16 @@ Found by the 2026-08 CLI audit (cluster <code>qualifier-rejected-elsewhere</code
       </article>
 
       <article class="cap rcard" id="audit-tap-dry-run-is-silently-ignored-outside-catalog-spec-and-def">
-        <div class="head"><span class="pill inflight">In flight</span><span class="cat">Safety · Bug</span></div>
+        <div class="head"><span class="pill shipped">Shipped</span><span class="cat">Safety · Bug</span></div>
         <h3><code>tap --dry-run</code> is silently ignored outside <code>--catalog</code>: SPEC and <code>--defaults</code> clone for real</h3>
+        <details class="cardbody"><summary>Write-up</summary>
         <p>On a pristine HOME, <code>boost tap --dry-run expo/skills</code> printed <code>&#10003; Tapped expo/skills (26 items)</code>, exit 0 in 1.56&nbsp;s &mdash; and the clone exists and <code>config.json</code> gained the row. <code>tap --defaults --dry-run</code> cloned five registries (<code>&#10003; tapped trailofbits/skills (124 items) &hellip;</code>) and grew the config from 21 to 26 taps. The multi-spec path is the same: <code>tap --dry-run expo/skills anthropics/skills</code> answered with <code>already tapped</code> lines, proving registry code ran. Help does say <em>&ldquo;with --catalog: print what would be tapped, tap nothing&rdquo;</em> &mdash; documented-narrow, but a flag literally named <code>--dry-run</code> that clones repos and writes config is a safety hole regardless of wording.
 
 The source confirms the shape: <code>taps.py</code> consults <code>args.dry_run</code> only inside <code>_tap_catalog</code> (<code>boost_cli/commands/taps.py:38</code>); <code>cmd_tap</code>'s defaults, single-spec and multi-spec branches (<code>taps.py:144-166</code>) never read it. Not a duplicate of the roadmap's <em>dry-run-promised-a-link-nobody-makes</em>, which is about <code>install --dry-run</code>.
 
 Fix, per the verified recommendation: in <code>cmd_tap</code>, when <code>args.dry_run</code> and (<code>args.spec</code> or <code>args.defaults</code>), print the parsed <code>(name, url)</code> per spec &mdash; or the <code>DEFAULT_TAPS</code> list &mdash; and return 0 <em>before</em> <code>registry.add</code>/<code>_tap_all</code>; update the <code>--dry-run</code> help text; add a unit test that <code>--dry-run SPEC</code> writes nothing. Docs: regenerate <code>docs/commands.html</code> after the help-text change.
 
-Found by the 2026-08 CLI audit (cluster <code>tap-dry-run-ignored</code>); repro in the audit log. Verified against source 2026-08-31.</p>
+Found by the 2026-08 CLI audit (cluster <code>tap-dry-run-ignored</code>); repro in the audit log. Verified against source 2026-08-31.</p></details>
         <div class="meta">
           <span class="m">Complexity <b>S</b></span>
           <span class="m hi">Impact <b>High</b></span>

{boost_skill_cli-1.2.82 → boost_skill_cli-1.2.83}/tests/unit/test_rag.py

@@ -12,6 +12,7 @@ from __future__ import annotations
 import json
 import math
 import os
+import sqlite3
 from pathlib import Path
 
 import pytest
@@ -1533,3 +1534,89 @@ class TestPostingsStore:
         rag._CACHE.clear()
         assert rag.ready() is False, \
             "a docs-only index would score every query to zero hits"
+
+
+class TestPostingsInterning:
+    """Terms live in their own table, not repeated on every posting row.
+
+    Measured on a real 458-tap store: 18,619,658 posting rows over 210,422
+    distinct terms averaging 6.6 characters — the term string alone was ~123 MB
+    of a 653 MB file, each term stored 88 times over on average. These tests
+    pin the interned shape directly against the SQLite file, because
+    `read_postings`/`stem_expansions` alone would pass just as well against the
+    old un-interned layout — a mutant that reverted `_write_postings` to store
+    `term TEXT` on every posting row would leave every other test in this file
+    green.
+    """
+
+    def _built(self):
+        rag._save([{"n": "a", "t": "x/y", "f": "a/SKILL.md", "h": "h1",
+                    "k": "skill", "c": 0, "l": 2, "snip": "sa",
+                    "tf": {"react": 2, "test": 1}},
+                   {"n": "b", "t": "x/y", "f": "b/SKILL.md", "h": "h2",
+                    "k": "skill", "c": 0, "l": 1, "snip": "sb",
+                    "tf": {"test": 3}}], {})
+        rag._CACHE.clear()
+
+    def _connect(self):
+        return sqlite3.connect("file:%s?mode=ro" % rag.postings_path(),
+                                uri=True)
+
+    def test_postings_rows_carry_an_integer_term_id_not_term_text(
+            self, sandbox):
+        self._built()
+        con = self._connect()
+        try:
+            cols = {row[1]: row[2]
+                    for row in con.execute("PRAGMA table_info(postings)")}
+        finally:
+            con.close()
+        assert cols == {"term_id": "INTEGER", "doc": "INTEGER", "tf": "INTEGER"}
+
+    def test_every_posting_row_resolves_to_a_term_via_the_terms_table(
+            self, sandbox):
+        self._built()
+        con = self._connect()
+        try:
+            rows = con.execute(
+                "SELECT t.term, p.doc, p.tf FROM postings p "
+                "JOIN terms t ON p.term_id = t.id "
+                "WHERE t.term = 'test' ORDER BY p.doc").fetchall()
+        finally:
+            con.close()
+        assert rows == [("test", 0, 1), ("test", 1, 3)]
+
+    def test_a_term_used_by_two_docs_is_stored_once_in_the_terms_table(
+            self, sandbox):
+        self._built()
+        con = self._connect()
+        try:
+            term_ids = {row[0] for row in
+                        con.execute("SELECT term_id FROM postings p "
+                                    "JOIN terms t ON p.term_id = t.id "
+                                    "WHERE t.term = 'test'")}
+            term_rows = con.execute(
+                "SELECT COUNT(*) FROM terms WHERE term = 'test'").fetchone()[0]
+        finally:
+            con.close()
+        assert len(term_ids) == 1, "test's two postings must share one term_id"
+        assert term_rows == 1, "the term string itself must appear exactly once"
+
+    def test_document_frequency_is_precomputed_per_term(self, sandbox):
+        self._built()
+        con = self._connect()
+        try:
+            df = dict(con.execute("SELECT term, df FROM terms"))
+        finally:
+            con.close()
+        assert df == {"react": 1, "test": 2}
+
+    def test_term_text_is_unique_in_the_terms_table(self, sandbox):
+        self._built()
+        con = sqlite3.connect(str(rag.postings_path()))
+        try:
+            with pytest.raises(sqlite3.IntegrityError):
+                con.execute("INSERT INTO terms (id, term, df) "
+                            "VALUES (999, 'test', 1)")
+        finally:
+            con.close()