boost-skill-cli 1.0.280__tar.gz → 1.0.282__tar.gz

{boost_skill_cli-1.0.280 → boost_skill_cli-1.0.282}/.gitignore

@@ -230,6 +230,11 @@ mutants/
 
 # sandboxed boost home for the Tier 1 eval corpus (`make eval` taps here)
 .eval-home/
+
+# `make evals` output. The generated corpus and index live in the system temp
+# dir (see evals/run_evals.py), so only the per-run scores land here; the
+# committed evals/baseline.json is what regressions are measured against.
+evals/results.json
 tests/visual/node_modules/
 tests/visual/out/
 tests/visual/package-lock.json

{boost_skill_cli-1.0.280 → boost_skill_cli-1.0.282}/.vale/styles/config/vocabularies/boost/accept.txt

@@ -34,6 +34,7 @@ mutmut
 mypy
 npm
 onboards
+overclaiming
 Pinecone
 pytest
 rebased

{boost_skill_cli-1.0.280 → boost_skill_cli-1.0.282}/Makefile

@@ -10,7 +10,7 @@ PYTEST    := $(VENV)/bin/pytest
 # pinned taps out of the developer's real ~/.boost.
 EVAL_HOME := $(CURDIR)/.eval-home
 
-.PHONY: venv test unit functional smoke coverage patch-coverage mutation lint check demo carousel clean-test eval eval-ai eval-rec eval-stats eval-explain audit dist-check bdd bench bench-cli fuzz post-deploy
+.PHONY: venv test unit functional smoke coverage patch-coverage mutation lint check demo carousel clean-test eval eval-ai eval-rec eval-stats eval-explain evals evals-baseline evals-golden evals-online audit dist-check bdd bench bench-cli fuzz post-deploy
 
 # Every tool comes from a hash-pinned requirements/*.txt — the same files CI
 # installs (see scripts/lock_toolchain.py). pip enforces the hashes, so a dev
@@ -52,7 +52,7 @@ mutation:
 	$(PY) scripts/mutation_gate.py --run --min 80
 
 lint:
-	$(VENV)/bin/ruff check boost_cli tests
+	$(VENV)/bin/ruff check boost_cli tests evals
 	$(VENV)/bin/mypy
 	$(VENV)/bin/pyright
 	$(VENV)/bin/lint-imports
@@ -72,6 +72,7 @@ lint:
 	$(PY) scripts/perf_gate.py
 	$(PY) scripts/check_anchors.py
 	$(PY) scripts/check_required_checks.py
+	$(PY) evals/make_golden.py --check
 
 # Supply-chain CVE gate: fail on a known OSV/PyPI advisory in the project's
 # dependency closure. Mirrors the pip-audit CI workflow; run before releasing.
@@ -125,6 +126,30 @@ eval-explain:
 	PYTHON=$(PY) BOOST_HOME=$(EVAL_HOME) bash scripts/ensure_eval_corpus.sh
 	BOOST_HOME=$(EVAL_HOME) $(PY) scripts/eval_explain.py --fail-under 0.80
 
+# Retrieval-quality gate over the HERMETIC golden set: recall@5/@10, MRR, and
+# nDCG@5/@10 with graded (3/2/1) relevance, plus a paired-bootstrap regression
+# test against the committed evals/baseline.json. Part of `check` and required
+# in CI. Unlike `eval` above it needs no network and no API key: it generates
+# its own corpus and taps it into a disposable BOOST_HOME under the system temp
+# dir, so a metric moving can only mean the ranker moved.
+evals:
+	$(PY) scripts/eval_gate.py --run
+
+# Re-pin evals/baseline.json to the current scores. Deliberate: it is how you
+# accept a ranking change, so it must be a separate commit that says why.
+evals-baseline:
+	$(PY) evals/run_evals.py --save-baseline
+
+# Regenerate evals/golden_set.json from evals/make_golden.py after editing the
+# query table (`--check` runs in `lint` and fails on drift).
+evals-golden:
+	$(PY) evals/make_golden.py
+
+# The same five metrics over the pinned REAL-registry corpus `eval` uses.
+# Needs the network; reported, never gated — mirrors `tests/smoke.sh --online`.
+evals-online:
+	$(PY) evals/run_evals.py --online --faithfulness
+
 # Opt-in Gherkin/BDD suite (tests/bdd) — additive to tests/functional, not a
 # replacement. Needs the [bdd] extra (`pip install -e '.[bdd]'`); not part of
 # `make check`.
@@ -183,7 +208,7 @@ demo:
 carousel:
 	for t in docs/carousel/tapes/*.tape; do vhs "$$t"; done
 
-check: lint eval test smoke mutation
+check: lint eval evals test smoke mutation
 	@echo "== all gates passed =="
 
 clean-test:

{boost_skill_cli-1.0.280 → boost_skill_cli-1.0.282}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: boost-skill-cli
-Version: 1.0.280
+Version: 1.0.282
 Summary: boost — Homebrew for AI coding skills
 Author: Jonathan Reyes
 License: GNU GENERAL PUBLIC LICENSE
@@ -997,16 +997,56 @@ reports, and the free services that monitor the project.
 
 ## Test suite
 
-Three layers, all enforced (`make check` runs the full set; CI runs the same thing):
+Four layers, all enforced (`make check` runs the full set; CI runs the same thing):
 
 | Layer | What it does | Gate |
 |---|---|---|
 | `make test` | pytest across `tests/unit/` (every core module) and `tests/functional/` (drives all 78 commands in-process against sandboxed homes) | **≥80% line coverage** of `boost_cli` (`fail_under` in pyproject.toml) |
 | `make smoke` | `tests/smoke.sh` — 170 checks run through the actual `./boost` shim (`--online` also hits real registries) | all pass |
 | `make mutation` | [mutmut] mutates `boost_cli/core` (~9,900 mutants) and reruns the unit suite against each one | **≥80% killed** (`scripts/mutation_gate.py`) |
+| `make evals` | scores the search ranker on a graded golden set — recall@5/@10, MRR, nDCG@5/@10 | **metric floors + no statistically significant regression** (`scripts/eval_gate.py`) |
 
 Dev setup: `make venv` (pulls in pytest, coverage, mutmut — the shipped
 runtime itself stays dependency-free). Every test run uses a throwaway
 `$HOME`, so your actual agent configs are never at risk.
 
 [mutmut]: https://mutmut.readthedocs.io/
+
+## Evaluation harness
+
+Search quality is the one thing the suite above can't assert: every test can
+pass while the ranker quietly gets worse. `make evals` measures it. The harness
+is pure stdlib and fully offline — it generates its own corpus of skills, rules,
+and workflows and taps it into a disposable `BOOST_HOME`, so a metric that moves
+can only mean the *ranker* moved. See [`evals/README.md`](evals/README.md).
+
+**Golden set** — [`evals/golden_set.json`](evals/golden_set.json): 36 realistic
+queries with **graded** relevance (3 perfect · 2 useful · 1 marginal) over the
+generated corpus. Generated and validated by `evals/make_golden.py`, which
+refuses a judgment naming a skill the corpus doesn't contain; `--check` fails CI
+on drift, so the file can never disagree with its source.
+
+**Five metrics**, all computed by `evals/metrics.py`:
+
+- **recall@5** / **recall@10** — of everything relevant, what fraction reached the top *k*.
+- **MRR** — 1/rank of the first relevant hit, averaged; sensitive only to the top of the list, which is where people look.
+- **nDCG@5** / **nDCG@10** — graded gain (`2^rel − 1`) with the standard `log2(rank + 1)` discount; the only metric that distinguishes the right results in the wrong order from the right order.
+
+**Significance gating.** A metric dropping isn't automatically a regression —
+with 36 queries, one unlucky result moves the mean. So `scripts/eval_gate.py`
+fails a metric only when it falls more than 0.02 below the committed baseline
+**and** a seeded 10,000-resample paired bootstrap over the per-query deltas puts
+that drop at `p < 0.05`. Alongside that, each metric has an absolute floor set
+~0.05 below its first measured value, which catches the erosion that a
+compare-to-last-commit check never sees. Accepting a real ranking change is a
+deliberate `make evals-baseline` commit.
+
+**Faithfulness.** `boost explain` and `boost distill` generate prose from a
+`SKILL.md`, and their failure mode is overclaiming. The harness scores that with
+the Ragas method — split the answer into atomic statements, verify each against
+the source, score supported/total — and reports it alongside what the runtime
+grounding guardrail in `boost explain` would do with the same reply. With no
+`claude` CLI and no `ANTHROPIC_API_KEY` it **skips rather than fails**, and
+reports a status instead of a number so "couldn't measure" is never mistaken for
+"scored zero". It's therefore outside `make check`; run it with
+`make evals-online`.

{boost_skill_cli-1.0.280 → boost_skill_cli-1.0.282}/README.md

@@ -281,16 +281,56 @@ reports, and the free services that monitor the project.
 
 ## Test suite
 
-Three layers, all enforced (`make check` runs the full set; CI runs the same thing):
+Four layers, all enforced (`make check` runs the full set; CI runs the same thing):
 
 | Layer | What it does | Gate |
 |---|---|---|
 | `make test` | pytest across `tests/unit/` (every core module) and `tests/functional/` (drives all 78 commands in-process against sandboxed homes) | **≥80% line coverage** of `boost_cli` (`fail_under` in pyproject.toml) |
 | `make smoke` | `tests/smoke.sh` — 170 checks run through the actual `./boost` shim (`--online` also hits real registries) | all pass |
 | `make mutation` | [mutmut] mutates `boost_cli/core` (~9,900 mutants) and reruns the unit suite against each one | **≥80% killed** (`scripts/mutation_gate.py`) |
+| `make evals` | scores the search ranker on a graded golden set — recall@5/@10, MRR, nDCG@5/@10 | **metric floors + no statistically significant regression** (`scripts/eval_gate.py`) |
 
 Dev setup: `make venv` (pulls in pytest, coverage, mutmut — the shipped
 runtime itself stays dependency-free). Every test run uses a throwaway
 `$HOME`, so your actual agent configs are never at risk.
 
 [mutmut]: https://mutmut.readthedocs.io/
+
+## Evaluation harness
+
+Search quality is the one thing the suite above can't assert: every test can
+pass while the ranker quietly gets worse. `make evals` measures it. The harness
+is pure stdlib and fully offline — it generates its own corpus of skills, rules,
+and workflows and taps it into a disposable `BOOST_HOME`, so a metric that moves
+can only mean the *ranker* moved. See [`evals/README.md`](evals/README.md).
+
+**Golden set** — [`evals/golden_set.json`](evals/golden_set.json): 36 realistic
+queries with **graded** relevance (3 perfect · 2 useful · 1 marginal) over the
+generated corpus. Generated and validated by `evals/make_golden.py`, which
+refuses a judgment naming a skill the corpus doesn't contain; `--check` fails CI
+on drift, so the file can never disagree with its source.
+
+**Five metrics**, all computed by `evals/metrics.py`:
+
+- **recall@5** / **recall@10** — of everything relevant, what fraction reached the top *k*.
+- **MRR** — 1/rank of the first relevant hit, averaged; sensitive only to the top of the list, which is where people look.
+- **nDCG@5** / **nDCG@10** — graded gain (`2^rel − 1`) with the standard `log2(rank + 1)` discount; the only metric that distinguishes the right results in the wrong order from the right order.
+
+**Significance gating.** A metric dropping isn't automatically a regression —
+with 36 queries, one unlucky result moves the mean. So `scripts/eval_gate.py`
+fails a metric only when it falls more than 0.02 below the committed baseline
+**and** a seeded 10,000-resample paired bootstrap over the per-query deltas puts
+that drop at `p < 0.05`. Alongside that, each metric has an absolute floor set
+~0.05 below its first measured value, which catches the erosion that a
+compare-to-last-commit check never sees. Accepting a real ranking change is a
+deliberate `make evals-baseline` commit.
+
+**Faithfulness.** `boost explain` and `boost distill` generate prose from a
+`SKILL.md`, and their failure mode is overclaiming. The harness scores that with
+the Ragas method — split the answer into atomic statements, verify each against
+the source, score supported/total — and reports it alongside what the runtime
+grounding guardrail in `boost explain` would do with the same reply. With no
+`claude` CLI and no `ANTHROPIC_API_KEY` it **skips rather than fails**, and
+reports a status instead of a number so "couldn't measure" is never mistaken for
+"scored zero". It's therefore outside `make check`; run it with
+`make evals-online`.

{boost_skill_cli-1.0.280 → boost_skill_cli-1.0.282}/boost_cli/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '1.0.280'
-__version_tuple__ = version_tuple = (1, 0, 280)
+__version__ = version = '1.0.282'
+__version_tuple__ = version_tuple = (1, 0, 282)
 
 __commit_id__ = commit_id = None

{boost_skill_cli-1.0.280 → boost_skill_cli-1.0.282}/boost_skill_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: boost-skill-cli
-Version: 1.0.280
+Version: 1.0.282
 Summary: boost — Homebrew for AI coding skills
 Author: Jonathan Reyes
 License: GNU GENERAL PUBLIC LICENSE
@@ -997,16 +997,56 @@ reports, and the free services that monitor the project.
 
 ## Test suite
 
-Three layers, all enforced (`make check` runs the full set; CI runs the same thing):
+Four layers, all enforced (`make check` runs the full set; CI runs the same thing):
 
 | Layer | What it does | Gate |
 |---|---|---|
 | `make test` | pytest across `tests/unit/` (every core module) and `tests/functional/` (drives all 78 commands in-process against sandboxed homes) | **≥80% line coverage** of `boost_cli` (`fail_under` in pyproject.toml) |
 | `make smoke` | `tests/smoke.sh` — 170 checks run through the actual `./boost` shim (`--online` also hits real registries) | all pass |
 | `make mutation` | [mutmut] mutates `boost_cli/core` (~9,900 mutants) and reruns the unit suite against each one | **≥80% killed** (`scripts/mutation_gate.py`) |
+| `make evals` | scores the search ranker on a graded golden set — recall@5/@10, MRR, nDCG@5/@10 | **metric floors + no statistically significant regression** (`scripts/eval_gate.py`) |
 
 Dev setup: `make venv` (pulls in pytest, coverage, mutmut — the shipped
 runtime itself stays dependency-free). Every test run uses a throwaway
 `$HOME`, so your actual agent configs are never at risk.
 
 [mutmut]: https://mutmut.readthedocs.io/
+
+## Evaluation harness
+
+Search quality is the one thing the suite above can't assert: every test can
+pass while the ranker quietly gets worse. `make evals` measures it. The harness
+is pure stdlib and fully offline — it generates its own corpus of skills, rules,
+and workflows and taps it into a disposable `BOOST_HOME`, so a metric that moves
+can only mean the *ranker* moved. See [`evals/README.md`](evals/README.md).
+
+**Golden set** — [`evals/golden_set.json`](evals/golden_set.json): 36 realistic
+queries with **graded** relevance (3 perfect · 2 useful · 1 marginal) over the
+generated corpus. Generated and validated by `evals/make_golden.py`, which
+refuses a judgment naming a skill the corpus doesn't contain; `--check` fails CI
+on drift, so the file can never disagree with its source.
+
+**Five metrics**, all computed by `evals/metrics.py`:
+
+- **recall@5** / **recall@10** — of everything relevant, what fraction reached the top *k*.
+- **MRR** — 1/rank of the first relevant hit, averaged; sensitive only to the top of the list, which is where people look.
+- **nDCG@5** / **nDCG@10** — graded gain (`2^rel − 1`) with the standard `log2(rank + 1)` discount; the only metric that distinguishes the right results in the wrong order from the right order.
+
+**Significance gating.** A metric dropping isn't automatically a regression —
+with 36 queries, one unlucky result moves the mean. So `scripts/eval_gate.py`
+fails a metric only when it falls more than 0.02 below the committed baseline
+**and** a seeded 10,000-resample paired bootstrap over the per-query deltas puts
+that drop at `p < 0.05`. Alongside that, each metric has an absolute floor set
+~0.05 below its first measured value, which catches the erosion that a
+compare-to-last-commit check never sees. Accepting a real ranking change is a
+deliberate `make evals-baseline` commit.
+
+**Faithfulness.** `boost explain` and `boost distill` generate prose from a
+`SKILL.md`, and their failure mode is overclaiming. The harness scores that with
+the Ragas method — split the answer into atomic statements, verify each against
+the source, score supported/total — and reports it alongside what the runtime
+grounding guardrail in `boost explain` would do with the same reply. With no
+`claude` CLI and no `ANTHROPIC_API_KEY` it **skips rather than fails**, and
+reports a status instead of a number so "couldn't measure" is never mistaken for
+"scored zero". It's therefore outside `make check`; run it with
+`make evals-online`.

{boost_skill_cli-1.0.280 → boost_skill_cli-1.0.282}/boost_skill_cli.egg-info/SOURCES.txt

@@ -173,6 +173,7 @@ docs/roadmap/items/ci-summary-step-fails-itself.md
 docs/roadmap/items/ci-time-is-now-runner-queueing.md
 docs/roadmap/items/clean-env-install-smoke-pip-and-pipx.md
 docs/roadmap/items/code-scanning-rule-can-be-restored.md
+docs/roadmap/items/codeql-job-rename-stranded-merge-protection.md
 docs/roadmap/items/command-carousel-page.md
 docs/roadmap/items/command-reference-docs-site.md
 docs/roadmap/items/complexity-and-dead-code-radar-xenon-vulture.md
@@ -326,6 +327,15 @@ docs/roadmap/items/workflow-linting-actionlint.md
 docs/roadmap/items/workflow-sast-zizmor.md
 docs/roadmap/items/workspace-scope-install.md
 docs/roadmap/items/zizmor-pin-is-a-yanked-release.md
+evals/README.md
+evals/__init__.py
+evals/baseline.json
+evals/faithfulness.py
+evals/golden_set.json
+evals/make_corpus.py
+evals/make_golden.py
+evals/metrics.py
+evals/run_evals.py
 examples/README.md
 examples/adapt-demo.sh
 examples/boost-run-prototype.sh
@@ -349,6 +359,7 @@ scripts/check_licenses.py
 scripts/check_required_checks.py
 scripts/ensure_eval_corpus.sh
 scripts/eval_explain.py
+scripts/eval_gate.py
 scripts/eval_recommend.py
 scripts/eval_retrieval.py
 scripts/eval_stats_summary.py
@@ -442,6 +453,7 @@ tests/unit/test_check_anchors.py
 tests/unit/test_check_licenses.py
 tests/unit/test_claude_settings.py
 tests/unit/test_cliparse.py
+tests/unit/test_codeql_analysis_key.py
 tests/unit/test_command_reference_fresh.py
 tests/unit/test_community_health_files.py
 tests/unit/test_config.py
@@ -453,6 +465,9 @@ tests/unit/test_docsite_chrome.py
 tests/unit/test_ed25519.py
 tests/unit/test_embed.py
 tests/unit/test_errors_and_cli_table.py
+tests/unit/test_eval_faithfulness.py
+tests/unit/test_eval_gate.py
+tests/unit/test_eval_metrics.py
 tests/unit/test_eval_stats_summary.py
 tests/unit/test_faithfulness.py
 tests/unit/test_frontmatter.py

boost_skill_cli-1.0.282/docs/roadmap/items/codeql-job-rename-stranded-merge-protection.md

@@ -0,0 +1,38 @@
+---
+id: codeql-job-rename-stranded-merge-protection
+board: code
+section: trust
+status: shipped
+category: CI · Bug
+complexity: S
+impact: High
+wow: 4
+note: a job rename forked the code-scanning config and blocked 100% of merges for a day
+order: 37
+owner: loop/codeql-analysis-key-guard
+pr:
+title: A CodeQL job rename silently blocked every merge
+---
+GitHub identifies a code-scanning configuration by <code>&lt;workflow path&gt;:&lt;job id&gt;</code>, which makes
+the job id load-bearing in a way nothing in the workflow file hints at: renaming it does not
+<i>move</i> the configuration, it <b>forks</b> it. #259 renamed this job <code>analyze</code> to
+<code>codeql-analyze</code> on 2026-07-27. The last <code>:analyze</code> analysis landed 20 seconds
+after that merge and never refreshed, stranding <b>247</b> of them on <code>refs/heads/main</code>
+where GitHub kept counting them as a configuration present on the base branch. Merge protection
+then reported <code>1 configuration not found</code> on every pull request — conclusion
+<code>neutral</code> — and because the branch ruleset carries a <code>code_scanning</code> rule,
+that neutral <b>blocked 100% of merges</b> while every required status check stayed green.
+The cutover is exact: the last <code>success</code> was #263, the first <code>neutral</code> #259,
+and every PR from #264 to #316 was neutral regardless of whether its diff contained Python at all.
+That symmetry is what makes it so easy to misread — it presents as a rule that rejects docs-only
+PRs, and the two hypotheses it invites (<i>"CodeQL has nothing to analyse"</i> and <i>"the rule is
+unsatisfiable"</i>) are both wrong. 29 docs-only PRs merged happily under the same rule before the
+rename. The check body says the real answer outright, and reading it beats inferring from
+<code>mergeable_state</code>. Fixed by deleting the 247 orphaned analyses, which was verified safe
+first: every alert still on the stale key was <code>fixed</code>, and all nine dismissed CodeQL
+alerts already carried their false-positive rationale on the live key. The job id was <i>already</i>
+guarded as a required status-check context by
+<code>scripts/check_required_checks.py</code>, and that guard passed during #259 because the
+context list was updated in the same commit — only the invisible half broke. So the guard added
+here pins the id from the analysis-key side, with a note that a future rename is not finished until
+the stale configuration is deleted. Related: [[release-verifies-the-wrong-commit]].

{boost_skill_cli-1.0.280 → boost_skill_cli-1.0.282}/docs/roadmap.html

@@ -110,7 +110,7 @@
       <div class="stat"><b>16</b><span>Shipped</span></div>
       <div class="stat"><b>1</b><span>Next up</span></div>
       <div class="stat"><b>4</b><span>Planned</span></div>
-      <div class="stat"><b>172</b><span>Loop finds</span></div>
+      <div class="stat"><b>173</b><span>Loop finds</span></div>
       <!-- /ROADMAP:stats -->
     </div>
   </section>
@@ -4014,6 +4014,40 @@ is built.</p>
         </div>
       </article>
 
+      <article class="cap rcard" id="codeql-job-rename-stranded-merge-protection">
+        <div class="head"><span class="pill shipped">Shipped</span><span class="cat">CI · Bug</span></div>
+        <h3>A CodeQL job rename silently blocked every merge</h3>
+        <p>GitHub identifies a code-scanning configuration by <code>&lt;workflow path&gt;:&lt;job id&gt;</code>, which makes
+the job id load-bearing in a way nothing in the workflow file hints at: renaming it does not
+<i>move</i> the configuration, it <b>forks</b> it. #259 renamed this job <code>analyze</code> to
+<code>codeql-analyze</code> on 2026-07-27. The last <code>:analyze</code> analysis landed 20 seconds
+after that merge and never refreshed, stranding <b>247</b> of them on <code>refs/heads/main</code>
+where GitHub kept counting them as a configuration present on the base branch. Merge protection
+then reported <code>1 configuration not found</code> on every pull request — conclusion
+<code>neutral</code> — and because the branch ruleset carries a <code>code_scanning</code> rule,
+that neutral <b>blocked 100% of merges</b> while every required status check stayed green.
+The cutover is exact: the last <code>success</code> was #263, the first <code>neutral</code> #259,
+and every PR from #264 to #316 was neutral regardless of whether its diff contained Python at all.
+That symmetry is what makes it so easy to misread — it presents as a rule that rejects docs-only
+PRs, and the two hypotheses it invites (<i>"CodeQL has nothing to analyse"</i> and <i>"the rule is
+unsatisfiable"</i>) are both wrong. 29 docs-only PRs merged happily under the same rule before the
+rename. The check body says the real answer outright, and reading it beats inferring from
+<code>mergeable_state</code>. Fixed by deleting the 247 orphaned analyses, which was verified safe
+first: every alert still on the stale key was <code>fixed</code>, and all nine dismissed CodeQL
+alerts already carried their false-positive rationale on the live key. The job id was <i>already</i>
+guarded as a required status-check context by
+<code>scripts/check_required_checks.py</code>, and that guard passed during #259 because the
+context list was updated in the same commit — only the invisible half broke. So the guard added
+here pins the id from the analysis-key side, with a note that a future rename is not finished until
+the stale configuration is deleted. Related: [[release-verifies-the-wrong-commit]].</p>
+        <div class="meta">
+          <span class="m">Complexity <b>S</b></span>
+          <span class="m hi">Impact <b>High</b></span>
+          <span class="m">Wow <b class="wow">★★★★</b></span>
+          <span class="m">a job rename forked the code-scanning config and blocked 100% of merges for a day</span>
+        </div>
+      </article>
+
       <article class="cap rcard" id="installed-skill-trust-audit">
         <div class="head"><span class="pill shipped">Shipped</span><span class="cat">Trust · Health</span></div>
         <h3><code>boost audit --skills</code> — a trust/staleness report for installed skills</h3>

boost_skill_cli-1.0.282/evals/README.md

@@ -0,0 +1,157 @@
+# evals — boost's offline evaluation harness
+
+A dev/CI quality gate for the retrieval stack, in the same family as mutation
+testing: `make evals`, never a `boost` subcommand. Pure stdlib, no network, no
+API key.
+
+```bash
+make evals              # run the harness and enforce the gate
+make evals-baseline     # deliberately re-pin evals/baseline.json
+make evals-golden       # regenerate golden_set.json after editing the queries
+make evals-online       # + the pinned real-registry corpus and faithfulness
+```
+
+## Why this exists alongside `make eval`
+
+The repo has two retrieval gates and they answer different questions.
+
+| | `make eval` | `make evals` |
+|---|---|---|
+| corpus | pinned GitHub repos (`tests/eval/taps.txt`) | generated locally, 57 items* |
+| network | required | none |
+| labels | binary | graded 3 / 2 / 1 |
+| metrics | recall@k, hit@1, MRR, nDCG@k | recall@5/@10, MRR, nDCG@5/@10 |
+| significance | `ranx` t-test, opt-in extra, informational | stdlib paired bootstrap, **gated** |
+| catches | "does a real skill come back for a real question" | "did the ranking get worse" |
+
+`eval` grades against the live ecosystem, so it drifts when upstream repos do —
+which is why its regression check is deliberately relaxed. This harness fixes
+the corpus so the only thing that can move a number is the ranker.
+
+\* 49 skills across 10 topic clusters, plus 4 rules and 4 workflows as ranking
+pressure. Confirm the total the way boost sees it:
+
+```bash
+BOOST_HOME=$TMPDIR/boost-evals-home boost count     # -> available 57 (across 1 tap)
+```
+
+## Layout
+
+| file | what it is |
+|---|---|
+| `metrics.py` | the arithmetic — recall@k, MRR, nDCG@k, paired bootstrap |
+| `make_corpus.py` | generates the corpus; **item table is the source of truth** |
+| `make_golden.py` | generates + validates `golden_set.json`; `--check` for drift |
+| `golden_set.json` | 36 queries, 107 graded judgments (generated — don't hand-edit) |
+| `baseline.json` | pinned per-query scores the bootstrap compares against |
+| `run_evals.py` | sets up the sandbox, scores every arm, writes `results.json` |
+| `faithfulness.py` | Ragas-methodology faithfulness for `explain` / `distill` |
+| `../scripts/eval_gate.py` | reads `results.json`, enforces the thresholds |
+
+The corpus and BM25 index are built into a disposable `BOOST_HOME` under the
+system temp dir (`$TMPDIR/boost-evals-home`, override with `BOOST_EVALS_HOME` or
+`--home`) — outside the repo on purpose, because it is itself a git repo and a
+nested one confuses `git status`, coverage discovery, and mutmut's tree copy.
+
+## The five metrics
+
+- **recall@5 / recall@10** — of everything graded relevant, what fraction landed
+  in the top k. Grades are binarized here (any grade ≥ 1 counts); recall asks
+  "did we find it", not "did we order it well".
+- **MRR** — 1/rank of the first relevant hit, averaged. Sensitive to the top of
+  the list only, which is where users actually look.
+- **nDCG@5 / nDCG@10** — DCG over ideal DCG, with the standard `log2(rank + 1)`
+  discount and exponential graded gains (`2^rel − 1`, so a perfect hit is worth
+  7 and a marginal one 1). This is the only metric that can tell a correct set
+  of results in the wrong order from the right order.
+
+## The grading scale
+
+| grade | meaning |
+|---|---|
+| 3 | perfect — the skill the query is asking for |
+| 2 | useful — a competent answer, not the best one |
+| 1 | marginal — topically adjacent; better than an unrelated hit |
+| — | absent from the label map, i.e. irrelevant |
+
+Grading the tail matters: a query labeled with only its perfect answer teaches
+nDCG nothing about the four results underneath it.
+
+## How the gate works
+
+`scripts/eval_gate.py` runs two independent checks and both must pass.
+
+1. **Absolute floors** — each metric must clear a floor calibrated ~0.05 below
+   the first measured run. This catches slow erosion across many merges, which a
+   compare-to-last-commit check never notices.
+2. **Significant regression vs the baseline** — a metric fails only when it drops
+   by more than `--regression-eps` (0.02) **and** a paired bootstrap over the
+   per-query deltas returns `p < 0.05`.
+
+Requiring both conditions is the whole design. The two arms answer the same 36
+queries, so pairing cancels out "this query is just hard" and leaves the effect
+of the change; resampling those differences 10,000 times (seeded, so the p-value
+is reproducible) asks how often a different draw of queries would have reversed
+the sign. A drop that is large but not significant is one unlucky query. A drop
+that is significant but tiny is a rounding change nobody should re-baseline for.
+Only a drop that is both is a regression.
+
+Accepting a genuine ranking change is `make evals-baseline` — its own commit,
+which says why.
+
+## Faithfulness
+
+`explain` and `distill` generate prose from a `SKILL.md`; their shared failure
+mode is overclaiming. Faithfulness measures it with the Ragas procedure —
+decompose the answer into atomic statements, verify each against the source,
+score supported/total — implemented directly on boost's `core.ai` bridge so the
+harness stays dependency-free and grades the same prompts the shipped commands
+use.
+
+Two numbers are reported per sample, and the gap between them is the point:
+
+| column | what it is |
+|---|---|
+| `judged` | the Ragas score over the **raw** model reply |
+| `runtime guard` | what `boost explain` would actually do with that reply |
+
+`boost explain` does not print whatever the model returns — `core/faithfulness.py`
+scores it for grounding and falls back to the deterministic extractive summary
+below a config-tunable threshold. Grading only the raw reply would answer a
+question nobody asks: how faithful is text the user may never see. The guard
+column costs no extra API calls (the sample already exists) and the run reports
+how many generations would have been discarded.
+
+The denominator is the number of statements **extracted**, never the number of
+verdicts returned. A judge reply is capped at `max_tokens`, so dividing by the
+verdicts would score 3/3 = 1.0 for an answer where 7 of 10 claims went
+unexamined — the metric would read best exactly when the judge coped worst. A
+short verdict list is a failed judging pass, not a score.
+
+It **skips, never fails**, when there is no `claude` CLI and no
+`ANTHROPIC_API_KEY`, and reports a status rather than a bare number so that
+"could not measure" is never mistakable for "scored 0.0". Because it needs a
+model it is not part of `make check`; run it via `make evals-online`.
+
+`--faithfulness-scorer ragas` cross-checks the stdlib scorer against the
+reference implementation by **delegating to `scripts/eval_explain.py`**, which
+owns the ragas path (`make eval-explain`, plus its own CI workflow). That script
+stays the ragas eval for the pinned real-registry corpus; this module is the
+hermetic, key-free counterpart that also covers `distill`. Delegating rather
+than copying is deliberate — an earlier draft duplicated its judge setup and
+silently dropped its `RunConfig` timeout.
+
+## Extending the golden set
+
+1. Add a row to `QUERIES` in `make_golden.py` with a fresh id, the query as a
+   user would actually type it, and a grade for **every** skill you would accept.
+2. Run `make evals-golden` and commit both files. `make_golden.py --check` runs
+   in `lint` and fails if they disagree.
+3. Adding queries invalidates the baseline — run `make evals-baseline` in the
+   same change and say so in the commit message.
+
+Add corpus items the same way, in `make_corpus.py`'s `SKILLS` table. Resist the
+temptation to reword a query because it scores badly: `q29` ("which alerts are
+worth paging someone for") currently misses entirely, because BM25 does not stem
+and the target skill says *alert*, not *alerts*. That is a real property of the
+shipped ranker, and an eval you tune until it looks good has stopped being one.

boost_skill_cli-1.0.282/evals/__init__.py

@@ -0,0 +1,6 @@
+"""boost's offline LLM evaluation harness (dev/CI only — never imported by the CLI).
+
+Run it with `make evals`, not through a `boost` subcommand: this is a quality
+gate in the same family as mutation testing, not user-facing functionality.
+See evals/README.md for the metric definitions and how to extend the golden set.
+"""