boost-skill-cli 1.2.75__tar.gz → 1.2.76__tar.gz

{boost_skill_cli-1.2.75/boost_skill_cli.egg-info → boost_skill_cli-1.2.76}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: boost-skill-cli
-Version: 1.2.75
+Version: 1.2.76
 Summary: boost — Homebrew for AI coding skills
 Author: Jonathan Reyes
 License-Expression: Apache-2.0

{boost_skill_cli-1.2.75 → boost_skill_cli-1.2.76}/boost_cli/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '1.2.75'
-__version_tuple__ = version_tuple = (1, 2, 75)
+__version__ = version = '1.2.76'
+__version_tuple__ = version_tuple = (1, 2, 76)
 
 __commit_id__ = commit_id = None

{boost_skill_cli-1.2.75 → boost_skill_cli-1.2.76}/boost_cli/commands/_common.py

@@ -8,7 +8,7 @@ place lets the modules split without either importing back into the other.
 """
 from __future__ import annotations
 
-from ..core import lockfile
+from ..core import lockfile, store
 from ..errors import BoostError
 
 
@@ -17,6 +17,34 @@ def _s(n: int) -> str:
     return "" if n == 1 else "s"
 
 
+def _require_lock_integrity() -> None:
+    """Fail loudly, before iterating, if the lock file itself is broken.
+
+    `lockfile.read()` collapses a missing/corrupt/wrong-schema lock into an
+    empty skeleton so ordinary reads degrade cleanly — but that means
+    `cmd_verify`/`cmd_drift` would iterate zero entries and report "nothing
+    installed" for exactly the state they exist to catch: a store with real
+    skills in it and no record of them. A missing lock over an empty store is
+    a fresh install, not a fault, so only that combination passes silently;
+    corrupt or wrong-schema is always reported, store empty or not, matching
+    `boost doctor`'s wording so the two commands never disagree.
+    """
+    integ = lockfile.check()
+    if integ.ok:
+        return
+    if integ.problem == "missing":
+        if not store.has_content():
+            return
+        raise BoostError(
+            "lock file missing — the store has skills but nothing is recorded",
+            hint="run `boost sync`")
+    if integ.problem == "corrupt":
+        raise BoostError("lock file is corrupt — restore with `boost replay`")
+    raise BoostError(
+        "lock file schema is v%s, expected v%d" % (integ.version, lockfile.SCHEMA_VERSION),
+        hint="restore with `boost replay`")
+
+
 def _iter_installed(names: list[str] | None = None) -> list[tuple[str, dict]]:
     """[(name, lock_entry)] — all installed skills, or the given names.
 

{boost_skill_cli-1.2.75 → boost_skill_cli-1.2.76}/boost_cli/commands/intelligence.py

@@ -208,6 +208,21 @@ def _distill_ai(new: str, sources: list[dict]) -> str | None:
     return text if meta.get("name") else None
 
 
+_FENCE_RE = re.compile(r"^`{3,}")
+_STRUCTURAL_LINES = frozenset({"```", "---", "}", ");"})
+_TABLE_RULE_RE = re.compile(r"^-{3,}$|^[\s:|-]*\|[\s:|-]*$")
+
+
+def _is_structural_line(key: str) -> bool:
+    """Pure-syntax lines that must survive a dedupe pass unconditionally.
+
+    A closing fence, a lone brace/paren, an hr, or a table separator row
+    repeats verbatim across sources by construction, not by accident —
+    deduping them corrupts the merged document's structure.
+    """
+    return key in _STRUCTURAL_LINES or bool(_TABLE_RULE_RE.match(key))
+
+
 def _distill_merge(new: str, sources: list[dict]) -> str:
     """Mechanical merge: union tags, dedupe exact-duplicate body lines."""
     tags: list[str] = []
@@ -222,17 +237,27 @@ def _distill_merge(new: str, sources: list[dict]) -> str:
             "version": "1.0.0"}
     if tags:
         meta["tags"] = tags
-    seen = set()
+    seen: set[str] = set()
     sections = ["# %s\n\nDistilled from: %s."
                 % (new, ", ".join(s["name"] for s in sources))]
     for s in sources:
         kept = []
+        in_fence = False
         for raw in s["body"].splitlines():
             key = raw.strip()
-            if key:
-                if key in seen:
-                    continue
-                seen.add(key)
+            if not key:
+                kept.append(raw)
+                continue
+            if _FENCE_RE.match(key):
+                in_fence = not in_fence
+                kept.append(raw)
+                continue
+            if in_fence or _is_structural_line(key):
+                kept.append(raw)
+                continue
+            if key in seen:
+                continue
+            seen.add(key)
             kept.append(raw)
         sections.append("## From %s\n\n%s" % (s["name"], "\n".join(kept).strip()))
     body = re.sub(r"\n{3,}", "\n\n", "\n\n".join(sections))

{boost_skill_cli-1.2.75 → boost_skill_cli-1.2.76}/boost_cli/commands/quality.py

@@ -42,7 +42,7 @@ from ..core import (
 )
 from ..core import output as out
 from ..errors import BoostError
-from ._common import _iter_installed, _iter_installed_all, _s
+from ._common import _iter_installed, _iter_installed_all, _require_lock_integrity, _s
 
 # --- conflict: normative-rule extraction -----------------------------------
 
@@ -405,20 +405,32 @@ def cmd_doctor(argv):
         out.info("no registries tapped — nothing is searchable yet; add the "
                  "recommended ones with `boost tap --defaults`", wrap=True)
 
-    lock_ok = True
-    lp = paths.lockfile_path()
-    if lp.exists():
-        try:
-            raw = json.loads(lp.read_text(encoding="utf-8"))
-            if raw.get("version") != lockfile.SCHEMA_VERSION:
-                bad("lock file schema is v%s, expected v%d"
-                    % (raw.get("version"), lockfile.SCHEMA_VERSION))
-                lock_ok = False
-        except (json.JSONDecodeError, OSError):
-            bad("lock file is corrupt — restore with `boost replay`")
-            lock_ok = False
-    if lock_ok:
+    # lockfile.read() collapses missing/corrupt/wrong-schema into an empty
+    # skeleton so ordinary reads degrade cleanly — but that is exactly why
+    # this used to print "lock file parses (v3)" for a lock file that did not
+    # exist. check() reports the raw state instead, so "parses" is only ever
+    # claimed for a file doctor actually parsed.
+    integ = lockfile.check()
+    if integ.ok:
         out.ok("lock file parses (v%d)" % lockfile.SCHEMA_VERSION)
+        lock_ok = True
+    elif integ.problem == "missing":
+        if store.has_content():
+            n = sum(1 for c in paths.store_dir().iterdir()
+                    if c.is_dir() and not c.name.startswith("."))
+            bad("lock file missing — %d store dir%s unrecorded, run `boost sync`"
+                % (n, _s(n)))
+            lock_ok = False
+        else:
+            out.info("no lock file yet — nothing installed")
+            lock_ok = True
+    elif integ.problem == "corrupt":
+        bad("lock file is corrupt — restore with `boost replay`")
+        lock_ok = False
+    else:  # "schema"
+        bad("lock file schema is v%s, expected v%d"
+            % (integ.version, lockfile.SCHEMA_VERSION))
+        lock_ok = False
 
     skills = lockfile.installed()
     enabled = agents.enabled_agents()
@@ -819,6 +831,10 @@ def cmd_drift(argv):
     ap.add_argument("--json", action="store_true")
     args = ap.parse_args(argv)
 
+    # Same rule as `boost verify`: a missing/corrupt/wrong-schema lock over a
+    # populated store is a fault, not "no skills installed".
+    _require_lock_integrity()
+
     rows = []
     for kind, name, entry in _iter_installed_all(args.names or None):
         status = (_drift_status(name, entry) if kind == "skill"

{boost_skill_cli-1.2.75 → boost_skill_cli-1.2.76}/boost_cli/commands/safety.py

@@ -36,7 +36,13 @@ from ..core import (
 from ..core import output as out
 from ..core import rules as rules_mod
 from ..errors import BoostError
-from ._common import _iter_installed, _iter_installed_all, _s, _shadowed_kinds
+from ._common import (
+    _iter_installed,
+    _iter_installed_all,
+    _require_lock_integrity,
+    _s,
+    _shadowed_kinds,
+)
 
 # --- audit: dangerous-content patterns ------------------------------------
 
@@ -335,6 +341,11 @@ def cmd_verify(argv):
     ap.add_argument("--json", action="store_true")
     args = ap.parse_args(argv)
 
+    # Before trusting anything the lock file says: a missing/corrupt/wrong-
+    # schema lock over a populated store must be reported, not read as "no
+    # skills installed" — that is the failure this command exists to catch.
+    _require_lock_integrity()
+
     # A named item may be installed at user OR project scope. Validate against
     # both up front so `boost verify <vendored-skill>` doesn't wrongly error
     # "not installed", and only hand user-scope names to the iterator. User

{boost_skill_cli-1.2.75 → boost_skill_cli-1.2.76}/boost_cli/core/lockfile.py

@@ -14,6 +14,7 @@ from __future__ import annotations
 import json
 import shutil
 from contextlib import suppress
+from typing import NamedTuple
 
 from . import paths, util
 
@@ -75,6 +76,41 @@ def _preserve_corrupt(p) -> None:
             "empty lock", p, backup or "(backup failed)")
 
 
+class Integrity(NamedTuple):
+    """The lock file's own health, as opposed to what it records.
+
+    ``read()`` collapses a missing, corrupt, or wrong-schema lock file into
+    an empty skeleton so every other caller can treat "never installed" and
+    "lock file broke" the same way when they only want the content. A health
+    check must not conflate the two: `boost verify`/`drift`/`doctor` call
+    this instead so they can tell a genuinely empty install apart from one
+    whose record vanished out from under a populated store.
+    """
+    ok: bool
+    problem: str | None    # None | "missing" | "corrupt" | "schema"
+    version: object = None  # the schema version actually found, for "schema"
+
+
+def check() -> Integrity:
+    """Report whether the lock file itself parses as the current schema.
+
+    Does not touch the store — a missing lock file with an empty store is a
+    fresh install, not a fault; callers that care about that distinction
+    combine this with a store-content check of their own.
+    """
+    p = paths.lockfile_path()
+    if not p.exists():
+        return Integrity(False, "missing")
+    try:
+        raw = json.loads(p.read_text(encoding="utf-8"))
+    except (json.JSONDecodeError, OSError):
+        return Integrity(False, "corrupt")
+    version = raw.get("version")
+    if version != SCHEMA_VERSION:
+        return Integrity(False, "schema", version)
+    return Integrity(True, None, version)
+
+
 def write(lock: dict) -> None:
     """Snapshot the existing lock to history, then write ``lock`` atomically.
 

{boost_skill_cli-1.2.75 → boost_skill_cli-1.2.76}/boost_cli/core/store.py

@@ -78,6 +78,22 @@ def installed() -> dict:
     return lockfile.installed()
 
 
+def has_content() -> bool:
+    """True if the canonical store holds any skill directory at all.
+
+    Independent of the lock file: a store dir survives even when
+    ``.skill-lock.json`` goes missing or corrupt, so this is how a caller
+    tells a genuinely fresh install (nothing installed, nothing to report)
+    apart from one whose lock record vanished out from under a populated
+    store (a fault `boost verify`/`drift`/`doctor` must not stay quiet
+    about).
+    """
+    root = paths.store_dir()
+    if not root.is_dir():
+        return False
+    return any(c.is_dir() and not c.name.startswith(".") for c in root.iterdir())
+
+
 def source_dir_for(entry: dict) -> Path:
     """Absolute path of a catalog entry's skill dir inside its tap clone.
 

{boost_skill_cli-1.2.75 → boost_skill_cli-1.2.76/boost_skill_cli.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: boost-skill-cli
-Version: 1.2.75
+Version: 1.2.76
 Summary: boost — Homebrew for AI coding skills
 Author: Jonathan Reyes
 License-Expression: Apache-2.0

{boost_skill_cli-1.2.75 → boost_skill_cli-1.2.76}/boost_skill_cli.egg-info/SOURCES.txt

@@ -503,6 +503,7 @@ docs/roadmap/items/property-based-tests-hypothesis-on-the-parsers.md
 docs/roadmap/items/prose-and-terminology-linting-vale.md
 docs/roadmap/items/prune-ignored-dirs-during-scan-dir-walk.md
 docs/roadmap/items/publish-gate-ignores-pip-audit-and-metadata.md
+docs/roadmap/items/publish-the-keyword-index.md
 docs/roadmap/items/publish-trigger-was-reachable-from-a-fork.md
 docs/roadmap/items/published-shards-have-no-consumer.md
 docs/roadmap/items/pytest-tmpdir-cve-blocked-by-py39-floor.md
@@ -555,6 +556,7 @@ docs/roadmap/items/shard-runs-reembed-everything-weekly.md
 docs/roadmap/items/shards-pipeline-never-produced-a-shard.md
 docs/roadmap/items/shareable-catalogue-bundle.md
 docs/roadmap/items/shift-left-gate-pre-commit-pre-commit-ci.md
+docs/roadmap/items/shrink-the-published-index.md
 docs/roadmap/items/single-imperative-rule-extractor.md
 docs/roadmap/items/single-tech-stack-prober.md
 docs/roadmap/items/smart-rerank-pays-per-novel-query.md
@@ -818,6 +820,7 @@ tests/unit/test_dense_status.py
 tests/unit/test_dense_status_cheap.py
 tests/unit/test_dense_vector_dedupe.py
 tests/unit/test_dependabot_config.py
+tests/unit/test_distill_merge.py
 tests/unit/test_docs_pages_linked.py
 tests/unit/test_docsite_chrome.py
 tests/unit/test_doctor_output_polish.py

{boost_skill_cli-1.2.75 → boost_skill_cli-1.2.76}/docs/roadmap/items/audit-catalog-findings.md

@@ -2,7 +2,7 @@
 id: audit-catalog-findings
 board: code
 section: dx
-status: inflight
+status: shipped
 category: CLI · Bug
 complexity: S
 impact: Med

{boost_skill_cli-1.2.75 → boost_skill_cli-1.2.76}/docs/roadmap/items/audit-distill-s-heuristic-merge-drops-repeated-fences-braces-writi.md

@@ -2,15 +2,15 @@
 id: audit-distill-s-heuristic-merge-drops-repeated-fences-braces-writi
 board: code
 section: dx
-status: planned
+status: shipped
 category: CLI · Bug
 complexity: S
 impact: High
 wow: 2
 note: merged SKILL.md keeps 4 fence lines where one source alone had 56 — blocks never close
 order: 205
-owner:
-pr:
+owner: loop/distill-merge-fence-corruption
+pr: 668
 title: "distill's heuristic merge drops repeated <code>```</code> fences/braces, writing a structurally corrupt SKILL.md"
 ---
 <code>_distill_merge</code> keeps a global <code>seen</code> set of every stripped non-blank line

{boost_skill_cli-1.2.75 → boost_skill_cli-1.2.76}/docs/roadmap/items/audit-verify-drift-say-nothing-installed-exit-0-and-doctor-says-lo.md

@@ -2,15 +2,15 @@
 id: audit-verify-drift-say-nothing-installed-exit-0-and-doctor-says-lo
 board: code
 section: health
-status: planned
+status: inflight
 category: CLI · Bug
 complexity: S
 impact: High
 wow: 2
 note: rm the lock, verify exits 0 'nothing installed'; doctor still prints 'lock parses (v3)'
 order: 214
-owner:
-pr:
+owner: loop/lock-integrity-reporting
+pr: 674
 title: "<code>verify</code>/<code>drift</code> say 'nothing installed' (exit 0) when the lock file is missing or corrupt"
 ---
 Delete <code>.skill-lock.json</code> after an install (store and four agent links still present) and

boost_skill_cli-1.2.76/docs/roadmap/items/publish-the-keyword-index.md

@@ -0,0 +1,68 @@
+---
+id: publish-the-keyword-index
+board: code
+section: planned
+status: planned
+category: Search · Performance
+complexity: L
+impact: High
+wow: 4
+note: every machine rebuilds 697 MB of index for a corpus that is byte-identical on all of them
+order: 99
+title: publish the keyword index the way vectors are published
+---
+Dense vectors are built once in CI and downloaded. The BM25 index is not:
+<code>core/rag.py</code> has <b>no export or import function at all</b>, and
+<code>shards.yml</code> / <code>scripts/publish_shards.py</code> are dense-only end to end. Every
+install rebuilds the same index from the same registries, at the same pinned commits, to produce
+the same bytes.
+
+<b>Measured, on a real 458-tap machine.</b> The on-disk index is
+<code>rag_index.json</code> <b>43.7&nbsp;MB</b> plus <code>rag_postings.sqlite</code>
+<b>653.0&nbsp;MB</b> — <b>696.7&nbsp;MB</b> for 18,619,658 postings. Build cost, timed over a
+9,306-entry / 69-tap slice: <b>4.54&nbsp;s</b> reading bodies and tokenizing, <b>3.83&nbsp;s</b>
+writing postings, <b>8.4&nbsp;s</b> total — about 0.9&nbsp;ms per entry, so roughly
+<b>65&nbsp;s</b> and <b>~900&nbsp;MB</b> extrapolated to the full 71,700-entry catalogue.
+
+<b>Which user actually pays it.</b> Not the default one: <code>boost quickstart</code> taps the
+<b>7 starter registries</b> and indexes them in about a second. The cost lands on
+<code>boost quickstart --catalog</code> — 463 registries, 2&nbsp;min 10&nbsp;s of parallel cloning
+and then a minute of indexing on top — and on anyone who taps their way there gradually.
+
+<b>The bug that makes this worth doing is not speed.</b> <code>boost catalog --import</code>
+already exists and already looks like the answer: <i>shareable-catalogue-bundle</i> advertises
+10.9&nbsp;MB replacing a 12&nbsp;GB clone and "59,972 searchable items in 4 seconds". That 4
+seconds is fast for a reason the card does not state. <code>rag.read_body</code> degrades
+<b>silently</b> to name + description when the item's clone is absent
+(<code>rag.py</code>: "Missing files degrade to just the catalog metadata"), and a bundle import
+restores catalogues with <i>zero repositories cloned</i>. So the index it builds is not the
+full-content index the <code>evals</code> gate floors — it is a frontmatter index wearing the same
+file name.
+
+<b>Measured directly</b> over 3,015 real entries, indexing them with and then without their
+clones: <b>3,041,326 tokens versus 182,507</b>. A bundle-only index carries <b>6.0%</b> of the
+searchable text, and nothing in the output says so. That is the same failure shape as an
+unpinned eval corpus — a number that still renders confidently while measuring something else.
+
+<b>Why this is easier than the dense shards, not harder.</b> BM25 looks like it needs global
+statistics, and it does — but none of them are frozen at build time. <code>_bm25</code> derives
+<code>n = len(docs)</code> and <code>df = len(plist)</code> on <i>every query</i>, so IDF is
+computed from whatever corpus is loaded. A per-registry shard therefore merges by offsetting
+<code>doc_id</code>, unioning the postings, and recomputing <code>avg_len</code> from per-shard
+totals — arithmetic, not re-derivation. And unlike vectors there is no embedding space to match
+and no API key to hold, so <code>shards.incompatible()</code> has no analogue here: a published
+keyword index is importable by everyone, including the keyless user who cannot use vectors at all.
+
+<b>Shape.</b> <code>rag.export_shard</code> / <code>rag.import_shard</code> mirroring
+<code>dense</code>'s pair, per-registry assets on the existing <code>shards-latest</code> release,
+rows carried in the same <code>manifest.json</code> with the same commit pin and sha256 — the
+carry-forward machinery in <code>publish_shards.py manifest --carry-forward</code> applies
+unchanged, because a registry whose commit did not move has an index that did not change either.
+Three invariants transfer verbatim from the dense side and each is load-bearing: verify before
+replacing, refuse a shard whose commit is not the tap's commit, and never treat a missing digest
+as a match.
+
+<b>The open question is payload size</b>, and it is large enough to be its own decision — see
+<a href="#shrink-the-published-index">shrink-the-published-index</a>. This card should not ship
+until that one has an answer, because publishing 697&nbsp;MB per refresh to save 65&nbsp;s of CPU
+is not obviously the right trade, and at the compressed sizes measured there it clearly is.

boost_skill_cli-1.2.76/docs/roadmap/items/shrink-the-published-index.md

@@ -0,0 +1,60 @@
+---
+id: shrink-the-published-index
+board: code
+section: planned
+status: planned
+category: Performance · Storage
+complexity: M
+impact: Med
+wow: 3
+note: 653 MB of postings holds 1.4 MB of distinct terms — the term string is stored 88 times over
+order: 100
+title: shrink the keyword index before publishing it — structure first, then compression
+---
+<a href="#publish-the-keyword-index">publish-the-keyword-index</a> is worth doing only if the
+artifact is small enough to ship weekly. This card is the measurement that decides it, and the
+first answer is that <b>compression is the second lever, not the first</b>.
+
+<b>What the format actually stores.</b> <code>_write_postings</code> creates
+<code>postings (term TEXT, doc INTEGER, tf INTEGER)</code> and inserts one row per posting, so the
+<i>term string is repeated in every row</i>. Measured on the real 458-tap store:
+<b>18,619,658</b> rows over <b>210,422</b> distinct terms averaging <b>6.6</b> characters. That is
+<b>~123&nbsp;MB of term text to carry 1.4&nbsp;MB of distinct term text — 88&times; redundancy</b>,
+before the per-row and B-tree overhead that turns it into a 653&nbsp;MB file (page_size 4096,
+167,174 pages, freelist 0, so it is not slack space).
+
+<b>Compression measured on that file, as it stands:</b>
+
+<b><code>rag_index.json</code></b> 43.7&nbsp;MB raw &middot; gzip&nbsp;-6 <b>10.6&nbsp;MB</b> (4.12&times;).<br>
+<b><code>rag_postings.sqlite</code></b> 653.0&nbsp;MB raw &middot; gzip&nbsp;-6 201.7&nbsp;MB (3.23&times;) &middot; zstd&nbsp;-3 169.9&nbsp;MB (3.84&times;) &middot; <b>zstd&nbsp;-19 106.7&nbsp;MB (6.11&times;)</b>.
+
+So even with no format change, <b>zstd -19 puts the whole index near 117&nbsp;MB</b> — under half
+the ~300&nbsp;MB of dense vectors already published weekly. The trade is already good; the point
+of this card is that it can be much better, and that the two levers compose.
+
+<b>Structure first, and it is the bigger win.</b> Interning terms into
+<code>terms(id, term)</code> with <code>postings(term_id, doc, tf)</code> removes ~123&nbsp;MB of
+duplicated strings <i>and</i> shrinks the <code>postings_term</code> index from a text key to an
+integer one. Beyond that, the classic inverted-index encodings apply directly because doc ids
+within a term are ascending: delta-encode them, varint or bitpack the deltas, and store one blob
+per term rather than one row per posting. Both shrink the file <b>on disk</b>, not just in
+transit, which is the half a compressed download never gives back — the user still ends up with
+653&nbsp;MB resident after import.
+
+<b>What must not regress.</b> <code>read_postings</code> exists precisely so a query touches a
+handful of terms instead of materialising the whole map — the change that took cold search from
+8-13&nbsp;s and multiple GB resident to 31-70&nbsp;ms of scoring. A blob-per-term layout keeps
+that property (one row read per query term, decoded on the spot); a scheme that requires decoding
+neighbouring terms to find one does not. <code>_bm25</code> must stay byte-identical, as it did
+through the SQLite move, and <code>TestBm25Math</code> is what says so.
+
+<b>Decompression cost is the thing to measure, not assume.</b> zstd -19 is slow to compress and
+fast to decompress, which is the right asymmetry for a weekly build feeding many imports — but
+"fast" needs a number on the import path before it is a claim, next to the 0.12&nbsp;s that
+importing dense rows costs today. A zstd dictionary trained across shards is the obvious follow-on
+for the many-small-registries case, where per-shard compression has little context to work with.
+
+<b>Deliverable.</b> A measured comparison — raw, interned, delta+varint, each &times; none/gzip/zstd
+— on the real store, with import-side decode time beside each. That table is what tells
+<a href="#publish-the-keyword-index">publish-the-keyword-index</a> what to ship, and it is worth
+having even if publishing is declined: the on-disk win applies to every install today.