boost-skill-cli 1.0.173__tar.gz → 1.0.175__tar.gz

{boost_skill_cli-1.0.173 → boost_skill_cli-1.0.175}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: boost-skill-cli
-Version: 1.0.173
+Version: 1.0.175
 Summary: boost — Homebrew for AI coding skills
 Author: Jonathan Reyes
 License: GNU GENERAL PUBLIC LICENSE

{boost_skill_cli-1.0.173 → boost_skill_cli-1.0.175}/boost_cli/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '1.0.173'
-__version_tuple__ = version_tuple = (1, 0, 173)
+__version__ = version = '1.0.175'
+__version_tuple__ = version_tuple = (1, 0, 175)
 
 __commit_id__ = commit_id = None

{boost_skill_cli-1.0.173 → boost_skill_cli-1.0.175}/boost_skill_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: boost-skill-cli
-Version: 1.0.173
+Version: 1.0.175
 Summary: boost — Homebrew for AI coding skills
 Author: Jonathan Reyes
 License: GNU GENERAL PUBLIC LICENSE

{boost_skill_cli-1.0.173 → boost_skill_cli-1.0.175}/boost_skill_cli.egg-info/SOURCES.txt

@@ -229,6 +229,7 @@ scripts/ensure_eval_corpus.sh
 scripts/eval_explain.py
 scripts/eval_recommend.py
 scripts/eval_retrieval.py
+scripts/eval_stats_summary.py
 scripts/import_budget.py
 scripts/mutation_gate.py
 style/README.md
@@ -286,6 +287,7 @@ tests/unit/test_dense_fallback.py
 tests/unit/test_docs_pages_linked.py
 tests/unit/test_embed.py
 tests/unit/test_errors_and_cli_table.py
+tests/unit/test_eval_stats_summary.py
 tests/unit/test_frontmatter.py
 tests/unit/test_gitleaks_config.py
 tests/unit/test_gitutil.py

{boost_skill_cli-1.0.173 → boost_skill_cli-1.0.175}/docs/roadmap/items/promote-nav-footer-into-the-shared-style-system.md

@@ -2,15 +2,15 @@
 id: promote-nav-footer-into-the-shared-style-system
 board: code
 section: dx
-status: planned
+status: shipped
 category: DX · Design system
 complexity: S
 impact: Med
 wow: 3
 note: <code>style/boost.css</code>
 order: 1
-owner:
-pr:
+owner: loop/theme
+pr: 197
 title: Promote nav / footer into the shared style system
 ---
 Canonical <code>style/boost.css</code> ships tokens + primitives but

{boost_skill_cli-1.0.173 → boost_skill_cli-1.0.175}/docs/roadmap/items/ranx-significance-monitor.md

@@ -2,15 +2,15 @@
 id: ranx-significance-monitor
 board: code
 section: health
-status: planned
+status: shipped
 category: Quality · Retrieval eval
 complexity: S
 impact: Med
 wow: 3
 note: p-value, not point delta
 order: 5
-owner:
-pr:
+owner: loop/evalstats
+pr: 198
 title: Significance-tracked engine comparison (ranx monitor)
 ---
 Tier 1b (the <code>ranx</code> paired t-test, <code>--stats</code>) only runs on

{boost_skill_cli-1.0.173 → boost_skill_cli-1.0.175}/docs/roadmap.html

@@ -1019,7 +1019,7 @@ harness.</p>
       </article>
 
       <article class="cap rcard">
-        <div class="head"><span class="pill planned">Planned</span><span class="cat">Quality · Retrieval eval</span></div>
+        <div class="head"><span class="pill shipped">Shipped</span><span class="cat">Quality · Retrieval eval</span></div>
         <h3>Significance-tracked engine comparison (ranx monitor)</h3>
         <p>Tier 1b (the <code>ranx</code> paired t-test, <code>--stats</code>) only runs on
 demand today. Wire it into a scheduled CI monitor — like the
@@ -1308,7 +1308,7 @@ improvement is real (or isn't)."</p>
 
       <!-- ROADMAP:cards section=dx -->
       <article class="cap rcard">
-        <div class="head"><span class="pill planned">Planned</span><span class="cat">DX · Design system</span></div>
+        <div class="head"><span class="pill shipped">Shipped</span><span class="cat">DX · Design system</span></div>
         <h3>Promote nav / footer into the shared style system</h3>
         <p>Canonical <code>style/boost.css</code> ships tokens + primitives but
            <strong>no nav, logo, or footer</strong>, so every page (the guide and

{boost_skill_cli-1.0.173 → boost_skill_cli-1.0.175}/scripts/eval_retrieval.py

@@ -18,9 +18,10 @@ the math, this asks "does the right skill actually come back for a real question
 
 Tier 1b (opt-in, offline): --stats runs a paired Student's t-test between the
 engines with `ranx`, so a metric gap is reported as statistically *significant*
-or not — a raw 0.919-vs-0.756 number can't say whether an engine is genuinely
-better or just luckier on 43 queries. Needs the [eval] extra; degrades cleanly
-if `ranx` is absent (never a core dependency).
+or not — a raw recall-vs-recall number can't say whether an engine is genuinely
+better or just luckier on the golden set. Needs the [eval] extra; degrades
+cleanly if `ranx` is absent (never a core dependency). A scheduled CI monitor
+(.github/workflows/eval-stats.yml) runs this and reports each metric's p-value.
 
 Tier 2a (LLM, opt-in, key-gated): --rerank measures the *lift* the LLM rerank
 stage (rag.rerank) adds over the raw BM25 order on the same golden set — no

boost_skill_cli-1.0.175/scripts/eval_stats_summary.py

@@ -0,0 +1,83 @@
+#!/usr/bin/env python3
+"""Render the `--stats --json` significance payload as a Markdown summary.
+
+Turns the ranx paired t-test output from ``eval_retrieval.py --stats --json``
+into a GitHub-Step-Summary table: for the best engine vs each other engine, is
+each metric's lead statistically significant (p < 0.05) or just noise. Kept
+separate from the workflow YAML so it is unit-testable (tests/unit) and so the
+significance-reading logic lives in one place.
+
+Usage:
+    python3 scripts/eval_stats_summary.py stats.json   # prints Markdown to stdout
+"""
+from __future__ import annotations
+
+import json
+import sys
+from typing import Optional
+
+ALPHA = 0.05
+HEADER = "### Tier 1b — retrieval significance (ranx paired t-test)\n"
+
+
+def _load(path: str) -> Optional[dict]:
+    try:
+        with open(path, encoding="utf-8") as fh:
+            return json.load(fh)
+    except (OSError, ValueError):
+        return None
+
+
+def render(payload: Optional[dict]) -> str:
+    """Markdown for a `--stats --json` payload (or a graceful skip note)."""
+    sig = (payload or {}).get("significance")
+    if not sig:
+        return (HEADER + "\n_skipped_ — no significance data "
+                "(the `[eval]` extra / `ranx` was unavailable, or fewer than "
+                "two engines ran).\n")
+
+    models = sig.get("model_names", [])
+    metrics = sig.get("metrics", [])
+    if len(models) < 2 or not metrics:
+        return HEADER + "\n_only one engine ran — nothing to compare._\n"
+
+    # The best engine is the last row of the comparison (engines are ordered
+    # worst→best upstream); every other engine carries the p-values of the best
+    # engine's lead over it in its `comparisons` block.
+    best = models[-1]
+    lines = [HEADER, "", "Best engine: **%s** (α = %.2f).\n" % (best, ALPHA)]
+    lines.append("| vs | " + " | ".join(metrics) + " |")
+    lines.append("|---|" + "|".join(["---"] * len(metrics)) + "|")
+
+    n_sig = 0
+    for other in models[:-1]:
+        comps = sig.get(other, {}).get("comparisons", {}).get(best, {})
+        cells = []
+        for m in metrics:
+            p = comps.get(m)
+            if p is None:
+                cells.append("—")
+            elif p < ALPHA:
+                cells.append("✅ p=%.4f" % p)
+                n_sig += 1
+            else:
+                cells.append("· p=%.4f" % p)
+        lines.append("| %s | %s |" % (other, " | ".join(cells)))
+
+    lines.append("")
+    lines.append("_✅ = %s beats that engine on the metric with p < %.2f "
+                 "(a statistically real lead, not just a higher number)._"
+                 % (best, ALPHA))
+    lines.append("\n%d of the compared metrics reach significance." % n_sig)
+    return "\n".join(lines) + "\n"
+
+
+def main(argv: Optional[list] = None) -> int:
+    args = list(sys.argv[1:] if argv is None else argv)
+    path = args[0] if args else "stats.json"
+    sys.stdout.write(render(_load(path)))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

{boost_skill_cli-1.0.173 → boost_skill_cli-1.0.175}/style/boost.css

@@ -326,6 +326,73 @@ a.cap, a.glass { text-decoration: none; }
   .stat  { padding: 18px 10px; }
 }
 
+/* ============================================================================
+   Site chrome — nav · logo · footer
+
+   The page frame, defined once. Every page used to re-declare this inline and
+   the Design Roadmap declared none at all, so its nav rendered unstyled.
+
+   These are a shared *floor*, not an override: every page links this sheet
+   BEFORE its own <style>, so a page still carrying inline chrome keeps
+   winning at equal specificity. Markup this expects:
+
+     <header>
+       <div class="wrap site-nav">            <!-- or: class="wrap nav" -->
+         <a class="logo" href="."><b>boost</b><span>Section</span></a>
+         <ul><li><a href="#x">X</a></li><li><a class="cta" href=".">Go</a></li></ul>
+       </div>
+     </header>
+   ========================================================================== */
+header {
+  position: sticky; top: 0; z-index: 50;
+  background: var(--panel-solid);
+  border-bottom: 1px solid var(--line);
+}
+
+.site-nav, header .nav {
+  display: flex; align-items: center; justify-content: space-between;
+  gap: 18px; padding: 12px 0;
+}
+
+.logo { display: flex; align-items: baseline; gap: 9px; text-decoration: none; }
+.logo b {
+  font-family: var(--mono); font-size: 18px; font-weight: 700;
+  color: var(--cyan); letter-spacing: -.02em;
+}
+.logo span {
+  font-size: 10.5px; letter-spacing: .22em; text-transform: uppercase;
+  color: var(--text-2);
+}
+
+.site-nav ul, header .nav ul {
+  display: flex; align-items: center; flex-wrap: wrap; gap: 18px;
+  list-style: none; margin: 0; padding: 0;
+}
+.site-nav a, header .nav a {
+  color: var(--text-2); text-decoration: none; font-size: 13.5px;
+}
+.site-nav a:hover, header .nav a:hover { color: var(--text); }
+
+/* the one emphasized link in the bar */
+.site-nav a.cta, header .nav a.cta {
+  color: var(--bg); background: var(--grad); font-weight: 700;
+  padding: 7px 14px; border-radius: 999px;
+}
+.site-nav a.cta:hover, header .nav a.cta:hover { color: var(--bg); opacity: .88; }
+
+footer {
+  margin-top: 64px; padding: 28px 0 40px;
+  border-top: 1px solid var(--line);
+  color: var(--text-3); font-size: 13px;
+}
+footer a { color: var(--text-2); text-decoration: none; }
+footer a:hover { color: var(--text); }
+
+@media (max-width: 720px) {
+  .site-nav, header .nav { flex-direction: column; align-items: flex-start; gap: 10px; }
+  .site-nav ul, header .nav ul { gap: 12px; }
+}
+
 /* ============================================================================
    Accessibility — honor reduced-motion, and print cleanly
    ========================================================================== */

boost_skill_cli-1.0.175/tests/unit/test_eval_stats_summary.py

@@ -0,0 +1,80 @@
+"""Unit tests for scripts/eval_stats_summary.py — the significance summary."""
+from __future__ import annotations
+
+import importlib.util
+import json
+from pathlib import Path
+
+import pytest
+
+_SCRIPT = Path(__file__).resolve().parents[2] / "scripts" / "eval_stats_summary.py"
+
+_skip = pytest.mark.skipif(not _SCRIPT.exists(), reason="script not reachable")
+
+
+def _mod():
+    spec = importlib.util.spec_from_file_location("eval_stats_summary", _SCRIPT)
+    assert spec and spec.loader
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+_PAYLOAD = {
+    "significance": {
+        "stat_test": "student",
+        "metrics": ["recall@10", "mrr"],
+        "model_names": ["catalog.search", "BM25 full-content"],
+        "catalog.search": {
+            "comparisons": {
+                "BM25 full-content": {"recall@10": 0.0047, "mrr": 0.20},
+            },
+        },
+        "BM25 full-content": {"comparisons": {}},
+    }
+}
+
+
+@_skip
+def test_significant_metric_marked_and_counted():
+    md = _mod().render(_PAYLOAD)
+    assert "Best engine: **BM25 full-content**" in md
+    assert "✅ p=0.0047" in md          # recall@10 is significant
+    assert "· p=0.2000" in md           # mrr is not
+    assert "1 of the compared metrics reach significance." in md
+
+
+@_skip
+def test_missing_significance_is_a_graceful_skip():
+    md = _mod().render({"k": 10})
+    assert "_skipped_" in md
+    assert "ranx" in md
+
+
+@_skip
+def test_none_payload_does_not_crash():
+    md = _mod().render(None)
+    assert "skipped" in md.lower()
+
+
+@_skip
+def test_single_engine_has_nothing_to_compare():
+    md = _mod().render({"significance": {
+        "metrics": ["recall@10"], "model_names": ["only-one"]}})
+    assert "nothing to compare" in md
+
+
+@_skip
+def test_main_reads_file_and_prints(tmp_path, capsys):
+    p = tmp_path / "stats.json"
+    p.write_text(json.dumps(_PAYLOAD), encoding="utf-8")
+    rc = _mod().main([str(p)])
+    assert rc == 0
+    assert "✅ p=0.0047" in capsys.readouterr().out
+
+
+@_skip
+def test_main_on_missing_file_is_graceful(capsys):
+    rc = _mod().main(["/no/such/stats.json"])
+    assert rc == 0
+    assert "skipped" in capsys.readouterr().out.lower()