boost-skill-cli 1.0.404__tar.gz → 1.0.405__tar.gz

{boost_skill_cli-1.0.404 → boost_skill_cli-1.0.405}/.lighthouserc.json

@@ -15,7 +15,7 @@
     "assert": {
       "aggregationMethod": "median-run",
       "assertions": {
-        "categories:performance": ["error", { "minScore": 0.8 }],
+        "categories:performance": ["error", { "minScore": 0.9 }],
         "categories:accessibility": ["error", { "minScore": 0.95 }],
         "categories:best-practices": ["error", { "minScore": 0.9 }],
         "categories:seo": ["error", { "minScore": 0.95 }]

{boost_skill_cli-1.0.404 → boost_skill_cli-1.0.405}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: boost-skill-cli
-Version: 1.0.404
+Version: 1.0.405
 Summary: boost — Homebrew for AI coding skills
 Author: Jonathan Reyes
 License: GNU GENERAL PUBLIC LICENSE

{boost_skill_cli-1.0.404 → boost_skill_cli-1.0.405}/boost_cli/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '1.0.404'
-__version_tuple__ = version_tuple = (1, 0, 404)
+__version__ = version = '1.0.405'
+__version_tuple__ = version_tuple = (1, 0, 405)
 
 __commit_id__ = commit_id = None

{boost_skill_cli-1.0.404 → boost_skill_cli-1.0.405}/boost_cli/commands/configuration.py

@@ -818,6 +818,28 @@ def cmd_serve(argv) -> int:
 REGISTRY = mcp.Registry()
 
 
+def _ranking_note(ranker: str) -> str:
+    """One line naming the ranking that actually produced this order.
+
+    `boost_search`'s own description promises an LLM rerank and quotes what it
+    buys — the right skill first 95% of the time against 79% without. When no
+    AI is configured that rerank degrades to the retrieval order, and the reply
+    was byte-for-byte the shape of a reranked one: same ten lines, same
+    confidence, 79%. An agent acts on the top result because the description
+    told it to.
+
+    `rag.rerank` already computes the only thing that distinguishes the two
+    cases, and its own comment says so — "the label is the only signal about
+    which engine" produced the order. This handler was discarding it.
+    """
+    if ranker == rag.LLM_RANKER:
+        return "\n(ranked by %s)" % ranker
+    return ("\n(ranked by %s — the LLM rerank named in this tool's description "
+            "did NOT run, so this is a shortlist to read rather than a verdict "
+            "to act on. Configure ANTHROPIC_API_KEY or the `claude` CLI to "
+            "enable it.)" % ranker)
+
+
 def _tool_search(args: dict):
     query = str(args.get("query", ""))
     rag.ensure()  # build the full-content index on first use (BM25 by default)
@@ -835,14 +857,16 @@ def _tool_search(args: dict):
     # default nobody chose. It degrades on its own when no AI is configured.
     rag_result = rag.search(query, limit=10, smart=True)
     if rag_result is not None:  # full-content index is built
-        hits, _ranker = rag_result
+        hits, ranker = rag_result
         if not hits:
             return "no skills match %r" % query, False
-        return "\n".join(
+        lines = [
             "%s — %s (%s)" % (h["entry"]["name"],
                               h["entry"].get("description", ""),
                               h["entry"]["tap"])
-            for h in hits), False
+            for h in hits]
+        lines.append(_ranking_note(ranker))
+        return "\n".join(lines), False
     # no index yet -> keep today's frontmatter search so nothing regresses
     scored = catalog.search(query)[:10]
     if not scored:

{boost_skill_cli-1.0.404 → boost_skill_cli-1.0.405}/boost_cli/core/rag.py

@@ -721,6 +721,13 @@ def retrieve(query: str, k: int = 60, kind: Optional[str] = None,
     return dedupe_by_content(hits, k)
 
 
+#: The label `rerank` returns when the LLM pass actually ran. Every other value
+#: is a retrieval engine's own name, i.e. "the rerank did not happen". Callers
+#: that quote what the rerank buys have to be able to tell those apart without
+#: string-matching a literal that could drift.
+LLM_RANKER = "Claude relevance"
+
+
 def rerank(query: str, hits: List[Hit], limit: int = 10,
            engine: str = "BM25 full-content") -> Tuple[List[Hit], str]:
     """Reorder the shortlist with the LLM bridge; degrade to ``engine`` order.
@@ -756,7 +763,7 @@ def rerank(query: str, hits: List[Hit], limit: int = 10,
     picked = [by_name[str(n)] for n in order if str(n) in by_name]
     rest = [h for h in hits if h["entry"]["name"]
             not in {str(n) for n in order}]
-    return (picked + rest)[:limit], "Claude relevance"
+    return (picked + rest)[:limit], LLM_RANKER
 
 
 # The damping constant from the original reciprocal-rank-fusion paper. It is

{boost_skill_cli-1.0.404 → boost_skill_cli-1.0.405}/boost_skill_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: boost-skill-cli
-Version: 1.0.404
+Version: 1.0.405
 Summary: boost — Homebrew for AI coding skills
 Author: Jonathan Reyes
 License: GNU GENERAL PUBLIC LICENSE

{boost_skill_cli-1.0.404 → boost_skill_cli-1.0.405}/boost_skill_cli.egg-info/SOURCES.txt

@@ -278,6 +278,7 @@ docs/roadmap/items/mcp-one-benefit-nameable-task.md
 docs/roadmap/items/mcp-register-names-server-before-env.md
 docs/roadmap/items/mcp-search-before-reinventing.md
 docs/roadmap/items/mcp-search-cost-was-understated.md
+docs/roadmap/items/mcp-search-hid-which-ranking-ran.md
 docs/roadmap/items/mcp-servers-ignore-install-scope.md
 docs/roadmap/items/mcp-startup-self-harden-fork-safety.md
 docs/roadmap/items/memoize-config-load-in-process.md

boost_skill_cli-1.0.405/docs/roadmap/items/mcp-search-hid-which-ranking-ran.md

@@ -0,0 +1,45 @@
+---
+id: mcp-search-hid-which-ranking-ran
+board: code
+section: internals
+status: shipped
+category: Correctness · MCP
+complexity: S
+impact: Medium
+wow: 4
+note: the degraded order was byte-for-byte the shape of the promised one
+order: 101
+owner: fix/mcp-search-names-its-ranker
+pr: 461
+title: boost_search never said which ranking produced its answer
+---
+<b>The MCP tool tells an agent an LLM reranks every match, "which is what makes the top result worth
+acting on rather than skimming ten". When no AI is configured, it doesn't &mdash; and the reply looked
+exactly the same.</b> Ten lines, same shape, same confidence, produced by BM25 alone. An agent acts on
+the top result because the description told it to.
+
+<b>The signal existed and was thrown away.</b> <code>rag.search</code> returns
+<code>(hits,&nbsp;ranker_label)</code>, and <code>rag.rerank</code>'s own docstring says why the label
+matters: "the label is the only signal about which engine answered, so a confident
+<code>BM25 full-content</code> sent debugging somewhere else." The MCP handler unpacked it into
+<code>_ranker</code> and dropped it on the floor. Every degrade path &mdash; no AI available, a reply
+that isn't a JSON array &mdash; already returns the retrieval label rather than claiming a rerank, so
+the distinction was fully computed and merely unreported.
+
+<b>This is not hypothetical on a normal machine.</b> The rerank needs
+<code>ANTHROPIC_API_KEY</code> or the <code>claude</code> CLI. A boost installed with
+<code>pipx</code> has neither inside its venv unless the key is exported, so the silent path is the
+common one rather than the edge case.
+
+<b>What shipped.</b> The reply now ends with the ranking that produced it &mdash;
+<i>"(ranked by Claude relevance)"</i> when the rerank ran, and otherwise a line naming the engine,
+saying plainly that the rerank <i>did not</i> run, and telling the caller to treat the order as a
+shortlist to read rather than a verdict to act on. <code>rag.LLM_RANKER</code> replaces the literal
+so the producer and the consumer of that label cannot drift apart, with a test driving
+<code>rerank</code> through all three of its outcomes.
+
+<b>A related claim was already corrected upstream.</b> The "95% against 79%" figures once quoted here
+were removed from both the tool description and the server instructions, because 0.791 was the BM25
+baseline over the <i>six</i>-repo corpus and the twenty-repo corpus that replaced it measures 0.4725
+&mdash; overstating the baseline by 31 points. The mechanism claim survived that edit, which is
+exactly the claim this card makes honest.

{boost_skill_cli-1.0.404 → boost_skill_cli-1.0.405}/docs/roadmap/items/roadmap-perf-budget-has-no-local-guard.md

@@ -84,3 +84,14 @@ raise to be a decision rather than an emergency.
 
 <b>Still genuinely open.</b> The page is not faster. 0.84 is still a board of 200 cards, and nothing
 here changes that &mdash; only the odds that the next step change is noticed the day it lands.
+
+<b>Corrected 2026-08-03: the page was never the slow part.</b> The <code>lighthouse</code> job served
+the docs with <code>python3 -m http.server</code>, which sends no <code>Content-Encoding</code>
+header at all, so every score above was measured on a document <b>3.27&times;</b> larger
+than the one GitHub Pages delivers. That is why both experiments here moved the number by
+<b>0.00</b>: each changed <i>layout</i> work while the score was being decided by <i>transfer</i>
+bytes. Compressing the harness the way Pages does took roadmap.html from
+<b>0.79&nbsp;&rarr;&nbsp;0.98</b> and FCP from <b>3,638&nbsp;&rarr;&nbsp;1,719&nbsp;ms</b>, and the
+floor went back to 0.90. The cheap local proxy this card called not obvious was markup bytes &mdash;
+the dimension <code>page_budget.py</code>, shipped by this very PR, already measures. See
+[[lighthouse-scored-a-page-nobody-is-served]].

{boost_skill_cli-1.0.404 → boost_skill_cli-1.0.405}/docs/roadmap.html

@@ -125,7 +125,7 @@
       <div class="stat"><b>16</b><span>Shipped</span></div>
       <div class="stat"><b>1</b><span>Next up</span></div>
       <div class="stat"><b>8</b><span>Planned</span></div>
-      <div class="stat"><b>217</b><span>Loop finds</span></div>
+      <div class="stat"><b>218</b><span>Loop finds</span></div>
       <!-- /ROADMAP:stats -->
     </div>
   </section>
@@ -3561,6 +3561,48 @@ written as a substring first and caught nothing until it was tightened.</p></det
           <span class="m">the bug was written down in a test comment and pinned as "behaves today"</span>
         </div>
       </article>
+
+      <article class="cap rcard" id="mcp-search-hid-which-ranking-ran">
+        <div class="head"><span class="pill shipped">Shipped</span><span class="cat">Correctness · MCP</span></div>
+        <h3>boost_search never said which ranking produced its answer</h3>
+        <details class="cardbody"><summary>Write-up</summary>
+        <p><b>The MCP tool tells an agent an LLM reranks every match, "which is what makes the top result worth
+acting on rather than skimming ten". When no AI is configured, it doesn't &mdash; and the reply looked
+exactly the same.</b> Ten lines, same shape, same confidence, produced by BM25 alone. An agent acts on
+the top result because the description told it to.
+
+<b>The signal existed and was thrown away.</b> <code>rag.search</code> returns
+<code>(hits,&nbsp;ranker_label)</code>, and <code>rag.rerank</code>'s own docstring says why the label
+matters: "the label is the only signal about which engine answered, so a confident
+<code>BM25 full-content</code> sent debugging somewhere else." The MCP handler unpacked it into
+<code>_ranker</code> and dropped it on the floor. Every degrade path &mdash; no AI available, a reply
+that isn't a JSON array &mdash; already returns the retrieval label rather than claiming a rerank, so
+the distinction was fully computed and merely unreported.
+
+<b>This is not hypothetical on a normal machine.</b> The rerank needs
+<code>ANTHROPIC_API_KEY</code> or the <code>claude</code> CLI. A boost installed with
+<code>pipx</code> has neither inside its venv unless the key is exported, so the silent path is the
+common one rather than the edge case.
+
+<b>What shipped.</b> The reply now ends with the ranking that produced it &mdash;
+<i>"(ranked by Claude relevance)"</i> when the rerank ran, and otherwise a line naming the engine,
+saying plainly that the rerank <i>did not</i> run, and telling the caller to treat the order as a
+shortlist to read rather than a verdict to act on. <code>rag.LLM_RANKER</code> replaces the literal
+so the producer and the consumer of that label cannot drift apart, with a test driving
+<code>rerank</code> through all three of its outcomes.
+
+<b>A related claim was already corrected upstream.</b> The "95% against 79%" figures once quoted here
+were removed from both the tool description and the server instructions, because 0.791 was the BM25
+baseline over the <i>six</i>-repo corpus and the twenty-repo corpus that replaced it measures 0.4725
+&mdash; overstating the baseline by 31 points. The mechanism claim survived that edit, which is
+exactly the claim this card makes honest.</p></details>
+        <div class="meta">
+          <span class="m">Complexity <b>S</b></span>
+          <span class="m">Impact <b>Medium</b></span>
+          <span class="m">Wow <b class="wow">★★★★</b></span>
+          <span class="m">the degraded order was byte-for-byte the shape of the promised one</span>
+        </div>
+      </article>
       <!-- /ROADMAP:cards -->
 
     </div>
@@ -7471,7 +7513,18 @@ from the other side: if the board ever creeps past 80% of its ceiling, it fails
 raise to be a decision rather than an emergency.
 
 <b>Still genuinely open.</b> The page is not faster. 0.84 is still a board of 200 cards, and nothing
-here changes that &mdash; only the odds that the next step change is noticed the day it lands.</p></details>
+here changes that &mdash; only the odds that the next step change is noticed the day it lands.
+
+<b>Corrected 2026-08-03: the page was never the slow part.</b> The <code>lighthouse</code> job served
+the docs with <code>python3 -m http.server</code>, which sends no <code>Content-Encoding</code>
+header at all, so every score above was measured on a document <b>3.27&times;</b> larger
+than the one GitHub Pages delivers. That is why both experiments here moved the number by
+<b>0.00</b>: each changed <i>layout</i> work while the score was being decided by <i>transfer</i>
+bytes. Compressing the harness the way Pages does took roadmap.html from
+<b>0.79&nbsp;&rarr;&nbsp;0.98</b> and FCP from <b>3,638&nbsp;&rarr;&nbsp;1,719&nbsp;ms</b>, and the
+floor went back to 0.90. The cheap local proxy this card called not obvious was markup bytes &mdash;
+the dimension <code>page_budget.py</code>, shipped by this very PR, already measures. See
+[[lighthouse-scored-a-page-nobody-is-served]].</p></details>
         <div class="meta">
           <span class="m">Complexity <b>S</b></span>
           <span class="m">Impact <b>Medium</b></span>

{boost_skill_cli-1.0.404 → boost_skill_cli-1.0.405}/tests/functional/test_cli_configuration.py

@@ -1011,6 +1011,12 @@ class TestMcp:
         # the flattering-but-stale claim the cost test below exists to prevent.
         # State the mechanism; leave the arithmetic to the eval gate.
         assert "95%" not in specs["boost_search"]
+
+        # The description still says the rerank happens ("ranks the matches
+        # with an LLM"), so the REPLY has to say when it did not — otherwise
+        # the degraded order is indistinguishable from the promised one. See
+        # TestSearchNamesItsRanker below.
+        assert "llm" in specs["boost_search"]
         # Stating the benefit must never quietly drop the cost with it.
         # A description whose job is to make a tool worth reaching for is the
         # one place a flattering lie discredits everything around it.
@@ -1469,3 +1475,69 @@ class TestSelfUpdate:
         ev = journal.events(action="self-update")[0]
         assert ev["subject"] == "9.9.9"          # git-derived, not the stale const
         assert ev["previous"] == __version__
+
+
+class TestSearchNamesItsRanker:
+    """The reply has to say which ranking produced it.
+
+    `boost_search`'s description tells the calling agent an LLM reranks every
+    match, "which is what makes the top result worth acting on rather than
+    skimming ten". With no AI configured that rerank silently degrades to the
+    retrieval order and the reply was byte-for-byte the shape of a reranked
+    one — same ten lines, same confidence. `rag.rerank` already computes the
+    only signal that distinguishes them; the handler discarded it.
+    """
+
+    def test_a_degraded_search_says_the_rerank_did_not_run(self, monkeypatch):
+        from boost_cli.commands import configuration
+        from boost_cli.core import rag
+        monkeypatch.setattr(
+            rag, "ensure", lambda *a, **k: True)
+        monkeypatch.setattr(rag, "search", lambda *a, **k: (
+            [{"entry": {"name": "x", "description": "d", "tap": "t"}}],
+            "BM25 full-content"))
+        text, _ = configuration._tool_search({"query": "anything"})
+        assert "did NOT run" in text
+        assert "BM25 full-content" in text
+        assert "ANTHROPIC_API_KEY" in text
+
+    def test_a_reranked_search_says_so_without_the_warning(self, monkeypatch):
+        from boost_cli.commands import configuration
+        from boost_cli.core import rag
+        monkeypatch.setattr(rag, "ensure", lambda *a, **k: True)
+        monkeypatch.setattr(rag, "search", lambda *a, **k: (
+            [{"entry": {"name": "x", "description": "d", "tap": "t"}}],
+            rag.LLM_RANKER))
+        text, _ = configuration._tool_search({"query": "anything"})
+        assert "(ranked by %s)" % rag.LLM_RANKER in text
+        assert "did NOT run" not in text
+
+    def test_rerank_returns_that_label_only_when_the_llm_answered(
+            self, monkeypatch):
+        """The producer and the consumer of the label must agree.
+
+        `rag.LLM_RANKER` exists so this is impossible to drift rather than
+        merely unlikely — the note in the MCP handler keys on it, and every
+        other value means "the rerank did not happen".
+        """
+        from boost_cli.core import ai, rag
+        hits = [{"entry": {"name": "b", "kind": "skill", "description": "d"},
+                 "snippet": ""},
+                {"entry": {"name": "a", "kind": "skill", "description": "d"},
+                 "snippet": ""}]
+
+        monkeypatch.setattr(ai, "available", lambda: False)
+        assert rag.rerank("q", hits, engine="BM25 full-content")[1] == \
+            "BM25 full-content"
+
+        monkeypatch.setattr(ai, "available", lambda: True)
+        monkeypatch.setattr(ai, "ask", lambda *a, **k: '["a", "b"]')
+        order, label = rag.rerank("q", hits, engine="BM25 full-content")
+        assert label == rag.LLM_RANKER
+        assert [h["entry"]["name"] for h in order] == ["a", "b"]
+
+        # The model replied, but not with an order — the label must fall back
+        # with the ordering, or it claims a rerank that did not take effect.
+        monkeypatch.setattr(ai, "ask", lambda *a, **k: "sorry, I cannot")
+        assert rag.rerank("q", hits, engine="BM25 full-content")[1] == \
+            "BM25 full-content"