boost-skill-cli 1.2.50__tar.gz → 1.2.51__tar.gz

{boost_skill_cli-1.2.50 → boost_skill_cli-1.2.51}/CLAUDE.md

@@ -534,6 +534,18 @@ Consequences for code you write here:
   the measured trade is **3.40 GB → 3.87 GB (+14%), in 1360 s**, peaking at
   ~2× while both copies coexist. Don't describe it as size-neutral — an earlier
   draft of this section did, from prediction rather than measurement.
+- **Embedding reports progress and commits as it goes.** `dense.build` takes
+  `on_progress(done, total)` and `_embed_and_store` commits every
+  `_COMMIT_EVERY` rows. Both are bug fixes, not polish: a full catalogue is
+  tens of thousands of distinct chunks under one fixed spinner label, which a
+  user reported as a hang after hours; and every row used to land in a single
+  transaction that committed after the last one, so an interrupt threw away the
+  whole build. Periodic commits are safe because `build` deletes each changed
+  tap's rows *before* re-inserting, so a partial store is replaced rather than
+  doubled. The total counts **distinct** texts, not rows — 42.9% of chunks on a
+  real install are repeats and are embedded once — so a progress total taken
+  from row count would over-report the work.
+
 - **Nothing on the search path may count `chunks`.** `SELECT COUNT(*)` scans the
   `chunks_tap` covering index — 8,419 pages / 34.5 MB, measured 1.94 s — and
   both `ready()` and `status()` did it on every search, the latter only to word

{boost_skill_cli-1.2.50/boost_skill_cli.egg-info → boost_skill_cli-1.2.51}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: boost-skill-cli
-Version: 1.2.50
+Version: 1.2.51
 Summary: boost — Homebrew for AI coding skills
 Author: Jonathan Reyes
 License-Expression: Apache-2.0

{boost_skill_cli-1.2.50 → boost_skill_cli-1.2.51}/boost_cli/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '1.2.50'
-__version_tuple__ = version_tuple = (1, 2, 50)
+__version__ = version = '1.2.51'
+__version_tuple__ = version_tuple = (1, 2, 51)
 
 __commit_id__ = commit_id = None

{boost_skill_cli-1.2.50 → boost_skill_cli-1.2.51}/boost_cli/commands/discovery.py

@@ -388,8 +388,8 @@ def cmd_reindex(argv):
         # holds — count, stats, trending, plain search. See the import-budget
         # gate (scripts/import_budget.py).
         from ..core import embed
-        with spin.Spinner("embedding chunks into the dense store"):
-            dense_stats = _reindex_dense(args.force)
+        with spin.Spinner("embedding chunks into the dense store") as sp:
+            dense_stats = _reindex_dense(args.force, spinner=sp)
     if args.as_json:
         print(json.dumps({"bm25": stats, "dense": dense_stats}
                          if args.dense else stats))
@@ -434,7 +434,44 @@ def cmd_reindex(argv):
     return 0
 
 
-def _reindex_dense(force):
+def _embed_progress(spinner):
+    """A callback that rewrites the spinner's label with real numbers.
+
+    Embedding is the longest thing boost does — tens of thousands of distinct
+    chunks at roughly a second each for a full catalogue — and it ran under a
+    fixed label for hours. "embedding chunks into the dense store" is
+    indistinguishable from a hang, which is exactly how it was reported.
+
+    The rate comes from this run rather than a constant, because the two
+    backends differ by an order of magnitude and a local model's throughput
+    depends on the machine. Until a batch has finished there is nothing to
+    estimate from, so it says the total and nothing else rather than guessing.
+    """
+    started = time.monotonic()
+
+    def report(done: int, total: int) -> None:
+        if spinner is None or not total:
+            return
+        pct = 100.0 * done / total
+        # The bar first, because it is the part that answers the question the
+        # user actually has — "is this moving?" — at a glance, before any of
+        # the digits are read. `spin.bar` is the same determinate bar the rest
+        # of the CLI draws, so this looks like boost rather than like a
+        # second progress convention.
+        label = "embedding %s %d%% · %s/%s chunks" % (
+            spin.bar(done, total, width=16), int(pct),
+            format(done, ","), format(total, ","))
+        elapsed = time.monotonic() - started
+        if done and elapsed > 1:
+            remaining = (total - done) * (elapsed / done)
+            if remaining >= 1:
+                label += " · ~%s left" % util.human_duration(remaining)
+        spinner.label = label
+
+    return report
+
+
+def _reindex_dense(force, spinner=None):
     """Build the dense vector store; returns stats or None when unavailable."""
     # Local import: the dense/embedding engines are opt-in (`search --dense`),
     # so they stay out of startup for every other command in this module.
@@ -447,7 +484,7 @@ def _reindex_dense(force):
     # only what changed. Doing it after would quantize, then write float32 rows
     # into a store that no longer has a float32 table.
     migrated = dense.quantize()
-    stats = dense.build(force=force)
+    stats = dense.build(force=force, on_progress=_embed_progress(spinner))
     if migrated and isinstance(stats, dict):
         stats = stats | {"quantized": migrated["chunks"]}
     return stats

{boost_skill_cli-1.2.50 → boost_skill_cli-1.2.51}/boost_cli/core/dense.py

@@ -25,6 +25,11 @@ from .rag import Hit, chunk, entry_key, read_body
 
 INDEX_VERSION = 2
 _BATCH = 128            # texts per embedding request
+
+#: Rows between commits while storing vectors. Small enough that an interrupted
+#: build keeps almost everything it embedded, large enough that the commits are
+#: not the cost — on a 750k-chunk store this is ~150 commits.
+_COMMIT_EVERY = 5000
 _POOL = 8              # chunk over-fetch factor for KNN before per-entry reduce
 
 
@@ -462,12 +467,20 @@ def ready() -> bool:
         con.close()
 
 
-def build(entries: list[dict] | None = None,
-          force: bool = False) -> dict | None:
+def build(entries: list[dict] | None = None, force: bool = False,
+          on_progress=None) -> dict | None:
     """(Re)embed and store chunk vectors, reusing unchanged taps.
 
     Returns stats, or ``None`` if the backend or an embeddings provider is
     unavailable (nothing is written in that case).
+
+    ``on_progress(done, total)`` is called before the first batch and after
+    each one. It exists because this is the longest-running thing boost does —
+    a full catalogue is tens of thousands of distinct chunks at roughly a
+    second each — and it used to run under a bare spinner that said "embedding
+    chunks into the dense store" for hours without a number. A user cannot
+    tell that apart from a hang, and the honest fix is to say how many there
+    are and how far in we got.
     """
     from .rag import _tap_commits, _tap_paths
     if not have_backend() or not embed.available():
@@ -517,7 +530,8 @@ def build(entries: list[dict] | None = None,
         removed_taps = sorted(_indexed_taps(con) - {e["tap"] for e in entries})
         _delete_taps(con, changed_taps + removed_taps)
 
-        added, failed_taps = _embed_and_store(con, fresh, tap_paths)
+        added, failed_taps = _embed_and_store(con, fresh, tap_paths,
+                                             on_progress=on_progress)
         # Only record a commit for a tap whose chunks actually landed. Recording
         # one for a tap that stored nothing makes every later non-forced run
         # treat it as already built and skip it — one transient rate limit would
@@ -743,8 +757,8 @@ def _delete_taps(con: sqlite3.Connection, taps: list[str]) -> None:
 
 
 def _embed_and_store(con: sqlite3.Connection, entries: list[dict],
-                     tap_paths: dict[str, Path] | None
-                     ) -> tuple[int, set]:
+                     tap_paths: dict[str, Path] | None,
+                     on_progress=None) -> tuple[int, set]:
     """Embed every chunk of ``entries``; returns ``(added, failed_taps)``.
 
     A batch the provider rejects (rate limit, quota, oversized input) yields no
@@ -778,15 +792,31 @@ def _embed_and_store(con: sqlite3.Connection, entries: list[dict],
             order.append(text)
 
     vec_of: dict[str, object] = {}
+    if on_progress is not None:
+        # Reported before the first request, so the size of the job is known
+        # up front rather than inferred from how long it has already taken.
+        on_progress(0, len(order))
     for start in range(0, len(order), _BATCH):
         batch = order[start:start + _BATCH]
         vecs = embed.embed(batch, input_type="document")
-        if not vecs or len(vecs) != len(batch):
-            continue     # taps are attributed per row below, not per batch
-        vec_of.update(zip(batch, vecs, strict=True))
+        if vecs and len(vecs) == len(batch):
+            vec_of.update(zip(batch, vecs, strict=True))
+        # else: taps are attributed per row below, not per batch — but the
+        # progress count still advances, or a run with a few rejected batches
+        # would appear to stall.
+        if on_progress is not None:
+            on_progress(min(start + _BATCH, len(order)), len(order))
 
     added = 0
     failed_taps: set = set()
+    # Durability, not speed. Every row used to land in one transaction that
+    # committed only after the last one, so interrupting a three-hour build —
+    # or losing the machine to a sleep or an OOM — threw away every vector it
+    # had paid for. Committing as the rows accumulate keeps the work, and
+    # re-running is still correct because `_delete_taps` has already removed
+    # each changed tap's old rows, so a partial store is replaced rather than
+    # doubled.
+    since_commit = 0
     for e, ci, text in rows:
         vec = vec_of.get(text)
         if vec is None:
@@ -801,6 +831,11 @@ def _embed_and_store(con: sqlite3.Connection, entries: list[dict],
              e.get("kind", "skill"), ci, text[:200].strip()))
         _store_vector(con, cur.lastrowid, mod.serialize_float32(vec))
         added += 1
+        since_commit += 1
+        if since_commit >= _COMMIT_EVERY:
+            con.commit()
+            since_commit = 0
+    con.commit()
     return added, failed_taps
 
 

{boost_skill_cli-1.2.50 → boost_skill_cli-1.2.51}/boost_cli/core/util.py

@@ -168,6 +168,23 @@ def rel_time(iso: str) -> str:
     return then.strftime("%Y-%m-%d")
 
 
+def human_duration(seconds: float) -> str:
+    """A short, honest duration: ``'45s'``, ``'12m'``, ``'2h 5m'``.
+
+    Coarse on purpose. This labels an estimate that is extrapolated from a
+    partial run, so seconds of precision on a two-hour figure would claim an
+    accuracy the number does not have.
+    """
+    seconds = max(0, int(seconds))
+    if seconds < 60:
+        return "%ds" % seconds
+    minutes = seconds // 60
+    if minutes < 60:
+        return "%dm" % minutes
+    hours, minutes = divmod(minutes, 60)
+    return "%dh %dm" % (hours, minutes) if minutes else "%dh" % hours
+
+
 def human_size(n: int) -> str:
     """Format a byte count for humans: ``1536`` -> ``'1.5KB'``.
 

{boost_skill_cli-1.2.50 → boost_skill_cli-1.2.51/boost_skill_cli.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: boost-skill-cli
-Version: 1.2.50
+Version: 1.2.51
 Summary: boost — Homebrew for AI coding skills
 Author: Jonathan Reyes
 License-Expression: Apache-2.0

{boost_skill_cli-1.2.50 → boost_skill_cli-1.2.51}/boost_skill_cli.egg-info/SOURCES.txt

@@ -682,6 +682,7 @@ tests/unit/test_dense.py
 tests/unit/test_dense_embed_dedupe.py
 tests/unit/test_dense_fallback.py
 tests/unit/test_dense_fix_hint.py
+tests/unit/test_dense_progress.py
 tests/unit/test_dense_quantized.py
 tests/unit/test_dense_ready_order.py
 tests/unit/test_dense_shard_real_schema.py

boost_skill_cli-1.2.51/tests/unit/test_dense_progress.py

@@ -0,0 +1,211 @@
+# Copyright the boost contributors.
+# SPDX-License-Identifier: Apache-2.0
+"""Embedding says how far along it is, and keeps what it embedded.
+
+`boost reindex --dense` is the longest thing boost does — a full catalogue is
+tens of thousands of distinct chunks at roughly a second each — and it ran under
+one fixed label, "embedding chunks into the dense store", for hours. That is
+indistinguishable from a hang, and it was reported as one.
+
+Worse, every row landed in a single transaction that committed only after the
+last one, so interrupting a three-hour build threw away everything it had paid
+to embed. Both halves are pinned here: progress is reported with real numbers
+from the first batch, and rows are durable long before the end.
+
+Re-running after an interrupt stays correct because `build` deletes each
+changed tap's rows before re-inserting them, so a partial store is replaced
+rather than doubled — asserted below, since that is what makes periodic commits
+safe rather than a duplication bug.
+"""
+from __future__ import annotations
+
+import sqlite3
+
+import pytest
+
+from boost_cli.core import dense, embed, rag, util
+
+
+def _vec_loadable() -> bool:
+    con = sqlite3.connect(":memory:")
+    try:
+        import sqlite_vec
+        con.enable_load_extension(True)
+        sqlite_vec.load(con)
+        con.enable_load_extension(False)
+        con.execute("create virtual table t using vec0(embedding float[4])")
+        return True
+    except Exception:
+        return False
+    finally:
+        con.close()
+
+
+# Scoped to the classes that need a vector store, not the module: the label
+# and duration tests are pure and must still run where the extra is absent —
+# which is every default install.
+needs_vec = pytest.mark.skipif(
+    not _vec_loadable(), reason="sqlite-vec extension not loadable here")
+
+
+def _toy_embed(texts, input_type="document"):
+    return [[float(len(t) % 7), 1.0, 0.0, 0.5] for t in texts]
+
+
+def _entries(n):
+    return [{"name": "s%03d" % i, "tap": "acme/skills", "kind": "skill",
+             "skill_md": "skills/s%03d/SKILL.md" % i} for i in range(n)]
+
+
+@pytest.fixture()
+def dense_env(sandbox, monkeypatch):
+    monkeypatch.setattr(embed, "embed", _toy_embed)
+    monkeypatch.setattr(embed, "provider", lambda: "openai")
+    monkeypatch.setattr(embed, "model", lambda: "toy-4")
+    monkeypatch.setattr(embed, "dimension", lambda: 4)
+    monkeypatch.setattr(embed, "available", lambda: True)
+    monkeypatch.setattr(rag, "_tap_paths", lambda: {"acme/skills": "/x"})
+    monkeypatch.setattr(rag, "_tap_commits", lambda: {"acme__skills": "c1"})
+    # One distinct body per entry, so the distinct-text count is the entry
+    # count and the progress totals are predictable.
+    monkeypatch.setattr(dense, "read_body", lambda e, tp=None: e["name"])
+    return monkeypatch
+
+
+@needs_vec
+class TestProgressReporting:
+    def test_it_reports_the_total_before_the_first_request(self, dense_env):
+        seen = []
+        dense.build(_entries(5), on_progress=lambda d, t: seen.append((d, t)))
+        # The size of the job has to be knowable up front; inferring it from
+        # how long it has already taken is what users were left doing.
+        assert seen[0] == (0, 5)
+
+    def test_it_reaches_the_total(self, dense_env):
+        seen = []
+        dense.build(_entries(5), on_progress=lambda d, t: seen.append((d, t)))
+        assert seen[-1] == (5, 5)
+
+    def test_progress_never_goes_backwards(self, dense_env):
+        seen = []
+        dense.build(_entries(300), on_progress=lambda d, t: seen.append(d))
+        assert seen == sorted(seen)
+
+    def test_it_advances_even_when_a_batch_is_rejected(self, dense_env):
+        # A provider that rejects a batch used to leave the count frozen for
+        # that stretch, which reads as the stall it is not.
+        dense_env.setattr(embed, "embed", lambda texts, input_type="document": [])
+        seen = []
+        dense.build(_entries(300), on_progress=lambda d, t: seen.append((d, t)))
+        assert seen[0] == (0, 300)
+        assert seen[-1] == (300, 300)
+
+    def test_no_callback_is_fine(self, dense_env):
+        # The MCP server and every non-interactive caller pass nothing.
+        assert dense.build(_entries(3)) is not None
+
+    def test_the_total_counts_distinct_texts_not_rows(self, dense_env):
+        # Registries mirror each other, so identical text is embedded once —
+        # a progress total that counted rows would over-report the work.
+        dense_env.setattr(dense, "read_body", lambda e, tp=None: "same body")
+        seen = []
+        dense.build(_entries(10), on_progress=lambda d, t: seen.append((d, t)))
+        assert seen[0] == (0, 1)
+
+
+@needs_vec
+class TestDurability:
+    def _rows(self):
+        con = sqlite3.connect(str(dense.db_path()))
+        try:
+            return con.execute("SELECT COUNT(*) FROM chunks").fetchone()[0]
+        finally:
+            con.close()
+
+    def test_rows_survive_an_interrupt_mid_build(self, dense_env):
+        # The failure this prevents: hours of embedding discarded because the
+        # single transaction never reached its commit.
+        boom = {"n": 0}
+        real = dense._store_vector
+
+        def flaky(con, rowid, blob):
+            boom["n"] += 1
+            if boom["n"] > dense._COMMIT_EVERY + 10:
+                raise KeyboardInterrupt("user pressed ctrl-c")
+            return real(con, rowid, blob)
+
+        dense_env.setattr(dense, "_store_vector", flaky)
+        dense_env.setattr(dense, "_COMMIT_EVERY", 20)
+        with pytest.raises(KeyboardInterrupt):
+            dense.build(_entries(200))
+        assert self._rows() > 0
+
+    def test_a_rerun_replaces_a_partial_store_rather_than_doubling_it(
+            self, dense_env):
+        dense.build(_entries(20))
+        first = self._rows()
+        dense.build(_entries(20), force=True)
+        # `build` deletes each changed tap's rows before re-inserting, which is
+        # what makes committing part-way through safe.
+        assert self._rows() == first
+
+
+class TestProgressLabel:
+    """What the user actually reads while it runs. No dense backend needed."""
+
+    class _Spin:
+        label = ""
+
+    def test_it_names_the_count_and_percentage(self):
+        from boost_cli.commands import discovery
+        sp = self._Spin()
+        discovery._embed_progress(sp)(12480, 62331)
+        assert "12,480/62,331 chunks" in sp.label
+        assert "20%" in sp.label
+
+    def test_it_draws_a_determinate_bar(self):
+        # The bar answers "is this moving?" before any digit is read, which is
+        # the question a fixed spinner left unanswerable for hours.
+        from boost_cli.commands import discovery
+        sp = self._Spin()
+        report = discovery._embed_progress(sp)
+        report(0, 100)
+        assert "░" in sp.label and "▓" not in sp.label
+        report(50, 100)
+        assert sp.label.count("▓") == 8      # half of a 16-wide bar
+        report(100, 100)
+        assert "░" not in sp.label
+
+    def test_no_estimate_before_there_is_anything_to_estimate_from(self):
+        # Until a batch has finished there is no rate; inventing one would be
+        # a guess dressed as a measurement.
+        from boost_cli.commands import discovery
+        sp = self._Spin()
+        discovery._embed_progress(sp)(0, 62331)
+        assert "left" not in sp.label
+
+    def test_a_missing_spinner_is_a_no_op(self):
+        from boost_cli.commands import discovery
+        discovery._embed_progress(None)(5, 10)      # must not raise
+
+    def test_zero_total_is_a_no_op(self):
+        from boost_cli.commands import discovery
+        sp = self._Spin()
+        discovery._embed_progress(sp)(0, 0)
+        assert sp.label == ""
+
+
+class TestHumanDuration:
+    """The estimate's wording: coarse, because the number is extrapolated."""
+
+    @pytest.mark.parametrize("seconds,want", [
+        (0, "0s"), (45, "45s"), (60, "1m"), (600, "10m"),
+        (3600, "1h"), (7500, "2h 5m"),
+    ])
+    def test_it_reads_as_a_person_would_say_it(self, seconds, want):
+        assert util.human_duration(seconds) == want
+
+    def test_negative_input_does_not_produce_nonsense(self):
+        # The estimate is (total - done) * rate; a clock that jumps backwards
+        # must not print "-3s left".
+        assert util.human_duration(-5) == "0s"