boost-skill-cli 1.0.306__tar.gz → 1.0.308__tar.gz

{boost_skill_cli-1.0.306/boost_skill_cli.egg-info → boost_skill_cli-1.0.308}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: boost-skill-cli
-Version: 1.0.306
+Version: 1.0.308
 Summary: boost — Homebrew for AI coding skills
 Author: Jonathan Reyes
 License: GNU GENERAL PUBLIC LICENSE

{boost_skill_cli-1.0.306 → boost_skill_cli-1.0.308}/boost_cli/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '1.0.306'
-__version_tuple__ = version_tuple = (1, 0, 306)
+__version__ = version = '1.0.308'
+__version_tuple__ = version_tuple = (1, 0, 308)
 
 __commit_id__ = commit_id = None

{boost_skill_cli-1.0.306 → boost_skill_cli-1.0.308}/boost_cli/commands/pkg.py

@@ -123,7 +123,33 @@ def _offer_mcp(res: store.InstallResult, no_mcp: bool = False) -> None:
     rows = res.mcp_servers
     if not rows or no_mcp or os.environ.get("BOOST_NO_MCP_OFFER"):
         return
-    from ..core import mcpdecl
+    from ..core import mcpdecl, scopes
+    # A project-scoped install already recorded these in the repo's .mcp.json
+    # (store.register_project_mcp), which is the committable, reviewable
+    # registration `--scope project` promises. Offering a `<host> mcp add` on
+    # top would register the same servers a second time, machine-wide — which
+    # is the bug this path had: the offer never looked at res.scope at all.
+    if res.scope == scopes.SCOPE_PROJECT:
+        # Report what was RECORDED, not what was declared. The two differ when
+        # the sidecar could not be written — a read-only checkout, a root-owned
+        # .mcp.json — and claiming success there would be the worse failure,
+        # because the skill really is installed and the servers really are not.
+        wanted = [r["name"] for r in mcpdecl.registrable(rows)]
+        if not wanted:
+            return
+        out.info("")
+        if res.mcp_recorded:
+            out.ok("recorded %s in %s"
+                   % (_plural(len(res.mcp_recorded), "MCP server"),
+                      mcpdecl.SIDECAR))
+            out.dim("  %s — commit it to share with the team"
+                    % ", ".join(res.mcp_recorded))
+        else:
+            out.warn("could not write %s — %s not registered"
+                     % (mcpdecl.SIDECAR, ", ".join(wanted)))
+            out.dim("  the skill is installed; fix the file's permissions and "
+                    "reinstall to record them")
+        return
     out.info("")
     out.warn("%s needs %s to work:" % (res.name, _plural(len(rows), "MCP server")))
     for row in rows:
@@ -142,22 +168,32 @@ def _offer_mcp(res: store.InstallResult, no_mcp: bool = False) -> None:
     targets = [h for h in mcphost.hosts() if shutil.which(mcphost.cli(h))] \
         or mcphost.hosts()
     where = " · ".join(mcphost.label(h) for h in targets)
-    if not out.confirm("register %s with %s now?"
+    # Name the scope. The prompt used to say only "register N servers with
+    # Claude Code now?", which is the one detail the answer turns on — this
+    # writes a registration every project on this machine will then launch.
+    if not out.confirm("register %s with %s, for every project on this machine?"
                        % (_plural(len(wirable), "server"), where), default=False):
         out.dim("  skipped — `%s mcp add …` when you're ready"
                 % mcphost.cli(targets[0]))
         return
     for row in wirable:
         for host in targets:
-            _register_mcp_server(row["name"], row["spec"], host=host)
+            _register_mcp_server(row["name"], row["spec"], host=host,
+                                 scope=res.scope)
 
 
-def _register_mcp_server(name: str, spec: dict, host: str = "claude") -> None:
-    """Run one `<host> mcp add` for a skill-declared server. Never raises."""
+def _register_mcp_server(name: str, spec: dict, host: str = "claude",
+                         scope: str = "user") -> None:
+    """Run one `<host> mcp add` for a skill-declared server. Never raises.
+
+    ``scope`` is threaded from the install rather than defaulted, because
+    ``register_argv``'s own default is ``"user"`` and silently taking it is how
+    a ``--scope project`` install ended up registering machine-wide.
+    """
     import subprocess
 
     from ..core import mcpdecl, mcphost
-    argv = mcpdecl.register_argv(name, spec, host=host)
+    argv = mcpdecl.register_argv(name, spec, host=host, scope=scope)
     if not shutil.which(mcphost.cli(host)):
         out.warn("`%s` CLI not found — run this yourself:" % mcphost.cli(host))
         out.info("  " + " ".join(argv))

{boost_skill_cli-1.0.306 → boost_skill_cli-1.0.308}/boost_cli/commands/quality.py

@@ -365,9 +365,25 @@ def cmd_doctor(argv):
     if not rotation:
         bad("journal is overdue for rotation — run `boost heal`")
 
+    # Which search engine will actually answer a query. Dense retrieval needs
+    # three things to line up and every one of them fails silently, so doctor
+    # is where the answer belongs — `search` only ever reports the engine that
+    # already ran, never that a configured one never got the chance.
+    _report_search_engine(bad)
+
     lp = logs.log_path()
     if lp.exists():
-        out.ok("diagnostic log at %s" % _tilde(lp))
+        # Existence is not health: a log the process cannot open makes every
+        # invocation print a PermissionError traceback from the logging module
+        # while this check happily reported a ✓ for the same file. Diagnostics
+        # are the first thing consulted when something else breaks, so a log
+        # that silently accepts nothing has to read as a fault.
+        if os.access(str(lp), os.W_OK):
+            out.ok("diagnostic log at %s" % _tilde(lp))
+        else:
+            bad("diagnostic log %s is not writable — every invocation is "
+                "failing to record; fix its permissions (chmod u+w)"
+                % _tilde(lp))
     crashes = sorted(paths.logs_dir().glob("crash-*.log")) \
         if paths.logs_dir().is_dir() else []
     if crashes:
@@ -391,6 +407,59 @@ def cmd_doctor(argv):
     return 1 if issues else 0
 
 
+# Why dense retrieval isn't serving, keyed by `dense.status()["reason"]`. Each
+# names the ONE next action; the reason order in `status()` guarantees only the
+# first missing link is ever reported, so these never chain.
+_DENSE_FIX = {
+    "no-backend": "install the extra: `pip install 'boost-skill-cli[rag]'`",
+    "no-key": "set VOYAGE_API_KEY or OPENAI_API_KEY",
+    "no-store": "build it: `boost reindex --dense`",
+    "version-changed": "rebuild it: `boost reindex --dense --force`",
+    "provider-changed": "rebuild it: `boost reindex --dense --force`",
+    "model-changed": "rebuild it: `boost reindex --dense --force`",
+    "dim-changed": "rebuild it: `boost reindex --dense --force`",
+    "empty": "rebuild it: `boost reindex --dense --force`",
+}
+
+
+def _report_search_engine(bad) -> None:
+    """Report the engine `boost search` will use, and why it isn't the best one.
+
+    Only a *degraded* dense tier counts against doctor's exit code: BM25 is the
+    documented default and most users never opt in, so an unconfigured tier is
+    healthy. Vectors already on disk that have stopped serving are not.
+    """
+    # Local import: the dense/embedding engines are opt-in and stay out of
+    # startup for every other command (scripts/import_budget.py enforces it).
+    from ..core import dense
+    st = dense.status()
+
+    if st["ready"]:
+        out.ok("semantic search active — %s %s (%d-d), %d chunk%s across %d tap%s"
+               % (st["provider"], st["model"], st["dim"] or 0,
+                  st["chunks"], _s(st["chunks"]), st["taps"], _s(st["taps"])))
+        return
+
+    fix = _DENSE_FIX.get(str(st["reason"]), "see `boost reindex --dense`")
+    if st["degraded"]:
+        # The store was built and is now dead weight: say what it holds, what
+        # changed, and that search has silently been on BM25 the whole time.
+        built = st["built_model"] or st["built_provider"] or "an older build"
+        detail = "built with %s" % built
+        if st["reason"] == "model-changed":
+            detail += ", live key is %s" % st["model"]
+        elif st["reason"] == "provider-changed":
+            detail += ", live key is %s" % st["provider"]
+        elif st["reason"] == "empty":
+            detail += " but holds no vectors"
+        bad("semantic search silently off — %d-chunk vector store %s; "
+            "searches are using BM25. %s" % (st["chunks"], detail, fix))
+        return
+
+    out.info("semantic search not configured — using the full-content BM25 "
+             "engine (%s)" % fix)
+
+
 def _print_skipped(skipped: List[dict]) -> None:
     """Note the rule/workflow entries `lint` passed over (they have no SKILL.md)."""
     if not skipped:

{boost_skill_cli-1.0.306 → boost_skill_cli-1.0.308}/boost_cli/core/dense.py

@@ -102,6 +102,106 @@ def _chunk_texts(entry: dict, tap_paths: Optional[Dict[str, Path]]
     return chunk(body) or [body]
 
 
+def _recorded_meta() -> dict:
+    """What an existing store says it was built with, read WITHOUT sqlite-vec.
+
+    Deliberately not routed through :func:`_connect`, which loads the extension
+    and returns ``None`` without it: the most useful diagnostic line ("your
+    store was built with voyage-3") would then vanish in exactly the case that
+    needs it — a user who dropped the ``[rag]`` extra and wants to know why
+    dense went quiet. ``meta`` is a plain table, so plain sqlite3 can read it.
+    """
+    p = db_path()
+    if not p.exists():
+        return {}
+    try:
+        con = sqlite3.connect(str(p))
+    except sqlite3.Error:
+        return {}
+    try:
+        meta = _read_meta(con)
+        try:
+            meta["_chunks"] = con.execute(
+                "SELECT COUNT(*) FROM chunks").fetchone()[0]
+        except sqlite3.DatabaseError:
+            meta["_chunks"] = 0
+        return meta
+    except sqlite3.DatabaseError:
+        # A truncated or non-sqlite file: report "no store", never raise into
+        # a health check whose whole job is to survive a broken install.
+        return {}
+    finally:
+        con.close()
+
+
+def status() -> dict:
+    """Why dense retrieval is, or is not, serving queries.
+
+    Pure inspection — never builds, never embeds, never needs the extra. Three
+    independent things must line up (the ``[rag]`` extra, an embeddings key, a
+    built store whose space matches the live one) and every one of them fails
+    *silently* today, because :func:`rag.retrieve_any` floors to BM25 and
+    returns. ``reason`` names which link is missing so a caller can say so.
+
+    ``degraded`` is the load-bearing distinction: a user who never configured
+    dense search is *healthy* (BM25 is the documented default), while a user
+    who did all three steps and is still on BM25 has a real problem no other
+    surface reports. Only the latter should move an exit code.
+    """
+    have_be = have_backend()
+    prov, mdl, dim = embed.provider(), embed.model(), embed.dimension()
+    meta = _recorded_meta()
+    b_prov = meta.get("provider")
+    b_mdl = meta.get("model")
+    b_dim = meta.get("dim")
+    b_ver = meta.get("version")
+    commits = meta.get("commits")
+    chunks = int(meta.get("_chunks") or 0)
+    store_exists = bool(meta)
+
+    # Order matters: report the *first* missing link, so the message names the
+    # next action rather than a downstream symptom of the same gap.
+    if not have_be:
+        reason: Optional[str] = "no-backend"
+    elif prov is None:
+        reason = "no-key"
+    elif not store_exists:
+        reason = "no-store"
+    elif b_ver != INDEX_VERSION:
+        reason = "version-changed"
+    elif b_prov != prov:
+        reason = "provider-changed"
+    elif b_mdl != mdl:
+        reason = "model-changed"
+    elif b_dim != dim:
+        reason = "dim-changed"
+    elif chunks <= 0:
+        reason = "empty"
+    else:
+        reason = None
+
+    # Unconfigured is not degraded; a built-but-unusable store is. Vectors on
+    # disk are the proof of intent — the user paid to embed them, so anything
+    # that stops them serving (a dropped extra, an unset key, a changed model)
+    # is a real fault. Without a store there is nothing to have regressed:
+    # "no-store" is an unfinished setup, and it is the one reason that implies
+    # store_exists is False, so this single clause covers every case.
+    degraded = store_exists and reason is not None
+
+    return {
+        "backend": have_be,
+        "provider": prov, "model": mdl, "dim": dim,
+        "store_exists": store_exists,
+        "built_provider": b_prov, "built_model": b_mdl,
+        "built_dim": b_dim, "built_version": b_ver,
+        "chunks": chunks,
+        "taps": len(commits) if isinstance(commits, dict) else 0,
+        "ready": reason is None,
+        "reason": reason,
+        "degraded": degraded,
+    }
+
+
 def ready() -> bool:
     """True when a usable, provider-matched vector index exists on disk."""
     if not have_backend() or not embed.available() or not db_path().exists():

{boost_skill_cli-1.0.306 → boost_skill_cli-1.0.308}/boost_cli/core/store.py

@@ -51,6 +51,10 @@ class InstallResult:
     # but never acted on: core stays non-interactive, so the command layer owns
     # the offer to register them — same split as `conflicts` and `score`.
     mcp_servers: List[dict] = field(default_factory=list)
+    # Names actually written to the project's .mcp.json — not what was
+    # declared. The two differ when the file could not be written, and the
+    # install report must show the former.
+    mcp_recorded: List[str] = field(default_factory=list)
 
 
 def skill_store_dir(name: str) -> Path:
@@ -266,6 +270,89 @@ def declared_mcp_servers(skill_dir) -> List[dict]:
     return mcpdecl.servers_for(meta, sidecar)
 
 
+def project_mcp_sidecar(base) -> Path:
+    """The repo's ``.mcp.json`` — where a project-scoped install records servers.
+
+    The committable file agents already read, which is the whole point of
+    project scope over shelling out to ``<host> mcp add``: a registration that
+    lives here is reviewable in a diff and arrives with a teammate's clone,
+    where one made through a host CLI exists only on the machine that ran it.
+    """
+    from . import mcpdecl
+    return Path(base) / mcpdecl.SIDECAR
+
+
+def _load_sidecar(path: Path) -> Optional[dict]:
+    """The sidecar document, or None if absent or unreadable.
+
+    Best-effort by design, exactly like :func:`declared_mcp_servers`: by the
+    time this runs the skill is already on disk, so a corrupt ``.mcp.json`` must
+    not turn a completed install into a traceback. ``merge_into`` treats a
+    non-dict as empty, so a damaged file is rebuilt rather than propagated.
+    """
+    import json
+    try:
+        return json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, ValueError):
+        return None
+
+
+def _write_sidecar(path: Path, doc: dict) -> bool:
+    """Write the sidecar as indented JSON. False if it could not be written.
+
+    Best-effort on the way OUT as well as in. ``_load_sidecar`` already refuses
+    to turn a corrupt file into a traceback, but the write is the half that runs
+    *after* the skill is materialized, the lock is written and the journal is
+    logged — so an unwritable ``.mcp.json`` (read-only checkout, a root-owned
+    file, a full disk) would crash an install that had already succeeded. The
+    caller reports what was actually recorded rather than what was declared, so
+    a failure here is visible instead of silent.
+    """
+    import json
+    try:
+        path.write_text(json.dumps(doc, indent=2, sort_keys=True) + "\n",
+                        encoding="utf-8")
+        return True
+    except OSError:
+        return False
+
+
+def register_project_mcp(base, rows, skill: str) -> List[str]:
+    """Record ``skill``'s declared MCP servers in the repo. Returns names added.
+
+    Never overwrites a server already present — one the user configured by hand,
+    or one another skill declared first — and never creates the file for a skill
+    that declares nothing registrable, so a repo gains an ``.mcp.json`` only
+    when there is genuinely something in it.
+    """
+    from . import mcpdecl as _decl
+    if not _decl.registrable(rows):
+        return []
+    path = project_mcp_sidecar(base)
+    doc, added = _decl.merge_into(_load_sidecar(path), rows, skill)
+    if added and not _write_sidecar(path, doc):
+        return []          # nothing was recorded; do not claim otherwise
+    return added
+
+
+def unregister_project_mcp(base, skill: str) -> List[str]:
+    """Remove the servers boost recorded for ``skill``. Returns names removed.
+
+    Marker-keyed entries only, so a hand-configured server and another skill's
+    server both survive. Writes nothing when it owns nothing, which keeps an
+    uninstall from rewriting — and reformatting — a file it has no claim on.
+    """
+    from . import mcpdecl as _decl
+    path = project_mcp_sidecar(base)
+    existing = _load_sidecar(path)
+    if existing is None:
+        return []
+    doc, removed = _decl.strip_owned(existing, skill)
+    if removed and not _write_sidecar(path, doc):
+        return []
+    return removed
+
+
 def _enforce_capability_policy(name: str, skill_md: Path) -> None:
     """Raise if the skill's declared/detected capabilities are denied by policy.
 
@@ -501,6 +588,12 @@ def _install_project_skill(entry: dict, force: bool = False,
     res.upgraded = existing is not None
     res.scope = scopes.SCOPE_PROJECT
     res.score, _ = util.score_skill(first)
+    res.mcp_servers = declared_mcp_servers(first)
+    # Project scope registers into the repo's own .mcp.json rather than leaving
+    # it to the command layer's `<host> mcp add` offer, which is user-scoped and
+    # would put a machine-wide registration behind a --scope project install.
+    # Recorded here so it lands with the skill and reverses with it.
+    res.mcp_recorded = register_project_mcp(resolved_base, res.mcp_servers, name)
     return res
 
 
@@ -525,8 +618,14 @@ def uninstall_project(name: str, base=None) -> dict:
         if m.get("agent"):
             removed.append(m["agent"])
     projectlock.remove_skill(resolved_base, name)
+    # Reverse exactly what the install recorded. Marker-keyed entries only, so a
+    # server the user configured by hand — or one another skill still needs —
+    # survives. Without this a skill tried once keeps launching its server for
+    # every project that clones the repo.
+    unregistered = unregister_project_mcp(resolved_base, name)
     journal.log("uninstall", name, scope=scopes.SCOPE_PROJECT)
     return {"name": name, "unlinked": removed, "entry": entry,
+            "mcp_unregistered": unregistered,
             "scope": scopes.SCOPE_PROJECT, "base": str(resolved_base)}
 
 

{boost_skill_cli-1.0.306 → boost_skill_cli-1.0.308/boost_skill_cli.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: boost-skill-cli
-Version: 1.0.306
+Version: 1.0.308
 Summary: boost — Homebrew for AI coding skills
 Author: Jonathan Reyes
 License: GNU GENERAL PUBLIC LICENSE

{boost_skill_cli-1.0.306 → boost_skill_cli-1.0.308}/boost_skill_cli.egg-info/SOURCES.txt

@@ -169,6 +169,7 @@ docs/roadmap/items/ambiguous-tap-short-name-resolution.md
 docs/roadmap/items/atomic-corruption-safe-lock-file-writes.md
 docs/roadmap/items/atomic-skill-install-temp-dir-swap.md
 docs/roadmap/items/autonomous-ship-workflow-and-isolated-worktree.md
+docs/roadmap/items/bm25-index-is-one-json-blob.md
 docs/roadmap/items/boost-run-live-agents.md
 docs/roadmap/items/bring-commands-under-mutation-testing.md
 docs/roadmap/items/broken-link-and-anchor-checking-lychee.md
@@ -201,6 +202,7 @@ docs/roadmap/items/docsite-chrome-and-content-audit.md
 docs/roadmap/items/docstring-coverage-interrogate.md
 docs/roadmap/items/doctor-checks-rules-and-workflows.md
 docs/roadmap/items/end-of-options-guard-on-git-commands.md
+docs/roadmap/items/eval-corpus-cannot-see-real-retrieval.md
 docs/roadmap/items/extension-free-tests-core-dense-py.md
 docs/roadmap/items/extract-mcp-http-servers-out-of-configuration-py.md
 docs/roadmap/items/finish-mutation-hardening-across-core.md
@@ -225,6 +227,7 @@ docs/roadmap/items/integrity-verification-boost-verify.md
 docs/roadmap/items/internal-architecture-diagrams.md
 docs/roadmap/items/invocation-pid-logging-for-crash-correlation.md
 docs/roadmap/items/journal-rotation-race-and-handle-leak.md
+docs/roadmap/items/keyless-dense-tier-local-static-embeddings.md
 docs/roadmap/items/keyless-item.md
 docs/roadmap/items/layering-guard-import-linter.md
 docs/roadmap/items/license-compliance-scanning.md
@@ -261,6 +264,7 @@ docs/roadmap/items/mutation-hardening-core-gitutil-py.md
 docs/roadmap/items/mutation-hardening-core-store-py.md
 docs/roadmap/items/mutation-shard-floor-is-one-file.md
 docs/roadmap/items/mypy-strict-mode-for-commands.md
+docs/roadmap/items/near-duplicate-items-eat-the-result-slots.md
 docs/roadmap/items/negative-limit-inverts-log-pulse-output.md
 docs/roadmap/items/onboard-overwrites-generated-files-without-confirm.md
 docs/roadmap/items/one-command-every-env-nox.md
@@ -274,6 +278,7 @@ docs/roadmap/items/pin-the-lint-toolchain.md
 docs/roadmap/items/post-deploy-always-defeated-by-skipped-setup.md
 docs/roadmap/items/post-deploy-smoke-headless-load-check.md
 docs/roadmap/items/pre-release-python-canary-3-14t-free-threaded.md
+docs/roadmap/items/prerequisites-and-semantic-search-setup.md
 docs/roadmap/items/project-scope-across-every-command.md
 docs/roadmap/items/project-scope-symlink-escape-on-write.md
 docs/roadmap/items/promote-nav-footer-into-the-shared-style-system.md
@@ -477,6 +482,7 @@ tests/unit/test_community_health_files.py
 tests/unit/test_config.py
 tests/unit/test_dense.py
 tests/unit/test_dense_fallback.py
+tests/unit/test_dense_status.py
 tests/unit/test_dependabot_config.py
 tests/unit/test_docs_pages_linked.py
 tests/unit/test_docsite_chrome.py
@@ -520,6 +526,7 @@ tests/unit/test_pkg_secrets.py
 tests/unit/test_platform_pins.py
 tests/unit/test_policy.py
 tests/unit/test_post_deploy_smoke.py
+tests/unit/test_project_mcp_sidecar.py
 tests/unit/test_projectlock.py
 tests/unit/test_provenance.py
 tests/unit/test_rag.py

boost_skill_cli-1.0.308/docs/roadmap/items/bm25-index-is-one-json-blob.md

@@ -0,0 +1,45 @@
+---
+id: bm25-index-is-one-json-blob
+board: code
+section: pipeline
+status: planned
+category: Search · Index
+complexity: L
+impact: High
+wow: 5
+note: 2.5 GB RSS per search at 50k items
+order: 72
+owner:
+pr:
+title: The BM25 index is one JSON blob, and it stops working between 10k and 50k items
+---
+<code>rag.py</code> persists the whole index as a single JSON file and
+<code>json.loads</code> it on every invocation. <code>_CACHE</code>
+(<code>rag.py:275</code>) is process-local, so a long-lived MCP server amortises
+the cost but <b>every cold <code>boost search</code> pays it in full</b>.
+Measured on a synthetic corpus of real repo prose, 10k items &rarr; 50k items:<br>
+index on disk 52.7 MB &rarr; <b>270 MB</b>;<br>
+<code>json.loads</code> per search 2.0&ndash;2.9 s &rarr; <b>12.2&ndash;13.6 s</b>;<br>
+peak RSS 702 MB &rarr; <b>2.49 GB</b>;<br>
+BM25 scoring itself 31 ms &rarr; 70 ms &mdash; cold start dominates by ~200&times;.
+On a real 83-tap install the index is already 132 MB at 11.1k entries. Scaling
+that shape to 50k gives ~594 MB and ~5.7 GB resident. <b>Target RSS, not
+bytes</b>: Python object overhead was measured at a consistent 9.5–9.6× file
+size across three independent runs, which is what OOM-kills an 8 GB machine
+while the disk figure still looks survivable.
+Two cheap wins come first and are worth landing on their own.
+<b>(1) Stop chunking.</b> An unchunked index with the item's surface
+(name + de-hyphenated name + description) counted alongside the body scored
+<code>recall@10 0.742 / hit@1 0.429</code> at <b>41 MB and 0.43 s</b> to load,
+versus the live chunked index's <code>0.720 / 0.407</code> at <b>132 MB and
+8–13 s</b> — better on every metric, 3.2× smaller, 20× faster to load.
+<b>(2) Get postings out of JSON.</b> FTS5 is compiled into CPython's bundled
+sqlite (verified: 3.53.4, <code>ENABLE_FTS5</code>), so there is a
+zero-dependency path — but probe it at runtime, because that is a per-build
+property, not a guarantee. A compact mmap-able binary postings format is the
+alternative. Either way <code>snip</code> text (46.7 MB of the 132 MB) belongs
+out of the hot path.
+Also note <code>build()</code>'s incremental path is O(entire corpus): reusing
+unchanged taps runs <code>_kept_docs</code> → <code>_postings_to_doc_tf</code>,
+which inverts every posting in the index — measured 35 s and 1.72 GB RSS at
+11.1k entries to reindex a single changed file.

boost_skill_cli-1.0.308/docs/roadmap/items/eval-corpus-cannot-see-real-retrieval.md

@@ -0,0 +1,42 @@
+---
+id: eval-corpus-cannot-see-real-retrieval
+board: code
+section: internals
+status: planned
+category: Quality · Eval
+complexity: M
+impact: High
+wow: 4
+note: gate reports 1.000 while users get 0.720
+order: 70
+owner:
+pr:
+title: The eval gate reports a perfect score on a corpus 4× smaller than a real user's
+---
+<code>make eval</code> reports BM25 <code>recall@10 = 1.000</code> against the
+pinned 23-tap corpus in <code>tests/eval/taps.txt</code>. Run the same script
+against a real 83-tap install and it reports <b>0.720</b> — below the gate's own
+0.85 floor — with <code>hit@1 0.407</code>, <code>MRR 0.526</code>,
+<code>nDCG 0.566</code>, and <code>rule</code> the weakest kind at 0.500 recall /
+0.286 hit@1. The gate has been advertising a perfect score while real-world
+recall sits at 0.72, and it is blind in three separate ways:
+<b>(1) recall-only flooring.</b> <code>--fail-under</code> gates
+<code>recall@k</code> and nothing else, so a total <code>hit@1</code> collapse
+from 0.780 to 0.000 would still pass.
+<b>(2) regression detection is switched off.</b> <code>Makefile</code> passes
+<code>--regression-eps 1</code>, which tolerates any absolute drop up to 1.0.
+<b>(3) a kind oracle.</b> Grading per-row by the golden set's <code>kind</code>
+field tells the retriever whether the answer is a skill, a rule or a workflow —
+which <code>boost search</code> never knows unless the user passes
+<code>--kind</code>. That single filter is worth +0.073 recall and +0.073 hit@1,
+enough to flatter any change measured against it.
+<b>The consequence is that no retrieval work in the repo is currently
+falsifiable</b>, which makes this a hard prerequisite for every other item in
+this group rather than a cleanup. Note the companion gate is in better shape:
+<code>make evals</code> (<code>scripts/eval_gate.py</code>) already floors five
+metrics and runs a paired bootstrap for significance — its limitation is corpus
+size (57 entries, 1 tap), not metric design, so the work is to grow an
+instrument that exists rather than to build one. Also needed: queries that
+<em>only</em> body text can satisfy. The golden set grades items by name, so it
+scores a surface-only index <em>above</em> the full-content one and would
+cheerfully greenlight deleting body indexing altogether.

boost_skill_cli-1.0.308/docs/roadmap/items/keyless-dense-tier-local-static-embeddings.md

@@ -0,0 +1,58 @@
+---
+id: keyless-dense-tier-local-static-embeddings
+board: code
+section: pipeline
+status: planned
+category: Search · Retrieval
+complexity: L
+impact: High
+wow: 5
+note: semantic search with no key, no billing, no deps
+order: 73
+owner:
+pr:
+title: Semantic search for users who will never set an API key
+---
+Dense retrieval today needs the <code>[rag]</code> extra <em>and</em> a
+<code>VOYAGE_API_KEY</code>/<code>OPENAI_API_KEY</code> <em>and</em> a built
+store. Most users will do none of that, so the default experience is BM25
+forever. The keyless path is a <b>local static embedding model</b> —
+<code>potion-retrieval-32M</code> class, MIT, model2vec family — which is not a
+transformer: the entire weight file is one lookup table, so inference is
+tokenize → gather rows → mean-pool → L2-normalize.
+Measured locally, pure stdlib: <b>~1 ms to embed a query</b> (mmap + bisect over
+sorted keys), 12.8 MB of int8 vectors for 50k items at 256-d, and <b>~20 ms to
+rerank BM25's top-200</b>. No numpy, no <code>sqlite-vec</code>, no ANN index,
+no new runtime dependency. <code>import numpy</code> alone costs 180–390 ms
+cold, which disqualifies it from a one-shot CLI query path; the BM25 prefilter is
+what makes the stdlib version viable, since a full 50k brute-force scan is 1.5 s
+in pure Python. Pool depth is justified by measurement: BM25 recall saturates at
+0.890 by depth 200 and gains nothing at 400, so reranking the top-200 gives up
+essentially nothing versus scanning everything.
+<b>Why this and not a shipped Voyage index.</b> A precomputed Voyage index is
+inert without a Voyage <em>query</em> vector, and the only keyless way to get one
+is a maintainer-run anonymous embedding endpoint — an unauthenticated free
+embeddings API backed by the maintainer's card, which also ships every user query
+off-machine and breaks offline. A local model is deterministic, so the artifact
+becomes a <em>cache</em> rather than a correctness dependency: a newly tapped
+repo can be embedded on the user's own machine. Doc-side is the asymmetry worth
+shipping for — 29 ms/doc in pure Python is ~24 min for 50k single-core, versus
+seconds in CI with numpy.
+<b>Do not ship this before the eval and dedup items.</b> The headline claim
+(+11.0 recall / +15.9 hit@1) did <em>not</em> survive verification: its baseline
+used the kind oracle the real search path lacks, and both the blend weight
+(<code>w_dense=0.7</code>) and the pool depth were argmax'd on the same 82
+queries they were reported on, by 2-query margins. On a binary metric at n=82
+the smallest net win reaching p&lt;0.05 is 6 queries; <code>hit@1</code> (+13 net
+queries) holds up, recall (+9) sits at the resolution floor. And the structural
+risk is real: the name is only ~10.5% of a mean-pooled surface vector while 106
+description clusters are shared across 270 distinct names, so the lift may
+<em>shrink</em> toward 50k rather than hold. Sequence: fix the gate, dedup, fix
+the index format, then re-measure with McNemar and a held-out blend weight,
+leading with <code>hit@1</code>. Test entry-level dense <em>alone</em> before the
+blend, ship 512-d not 256-d (the whole case for 256-d was one query), keep
+Voyage/OpenAI as the opt-in ceiling, and keep BM25 as the floor. The model table
+cannot go in the default wheel — the shipped runtime is 0.79 MB and every merge
+to main cuts a release, so +17.4 MB × ~24 releases/day exhausts PyPI's 10 GB
+project quota in under a month; it needs a separate, rarely-released data
+package behind an extra.