boost-skill-cli 1.0.174__tar.gz → 1.0.175__tar.gz

{boost_skill_cli-1.0.174 → boost_skill_cli-1.0.175}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: boost-skill-cli
-Version: 1.0.174
+Version: 1.0.175
 Summary: boost — Homebrew for AI coding skills
 Author: Jonathan Reyes
 License: GNU GENERAL PUBLIC LICENSE

{boost_skill_cli-1.0.174 → boost_skill_cli-1.0.175}/boost_cli/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '1.0.174'
-__version_tuple__ = version_tuple = (1, 0, 174)
+__version__ = version = '1.0.175'
+__version_tuple__ = version_tuple = (1, 0, 175)
 
 __commit_id__ = commit_id = None

{boost_skill_cli-1.0.174 → boost_skill_cli-1.0.175}/boost_skill_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: boost-skill-cli
-Version: 1.0.174
+Version: 1.0.175
 Summary: boost — Homebrew for AI coding skills
 Author: Jonathan Reyes
 License: GNU GENERAL PUBLIC LICENSE

{boost_skill_cli-1.0.174 → boost_skill_cli-1.0.175}/boost_skill_cli.egg-info/SOURCES.txt

@@ -229,6 +229,7 @@ scripts/ensure_eval_corpus.sh
 scripts/eval_explain.py
 scripts/eval_recommend.py
 scripts/eval_retrieval.py
+scripts/eval_stats_summary.py
 scripts/import_budget.py
 scripts/mutation_gate.py
 style/README.md
@@ -286,6 +287,7 @@ tests/unit/test_dense_fallback.py
 tests/unit/test_docs_pages_linked.py
 tests/unit/test_embed.py
 tests/unit/test_errors_and_cli_table.py
+tests/unit/test_eval_stats_summary.py
 tests/unit/test_frontmatter.py
 tests/unit/test_gitleaks_config.py
 tests/unit/test_gitutil.py

{boost_skill_cli-1.0.174 → boost_skill_cli-1.0.175}/docs/roadmap/items/ranx-significance-monitor.md

@@ -2,15 +2,15 @@
 id: ranx-significance-monitor
 board: code
 section: health
-status: planned
+status: shipped
 category: Quality · Retrieval eval
 complexity: S
 impact: Med
 wow: 3
 note: p-value, not point delta
 order: 5
-owner:
-pr:
+owner: loop/evalstats
+pr: 198
 title: Significance-tracked engine comparison (ranx monitor)
 ---
 Tier 1b (the <code>ranx</code> paired t-test, <code>--stats</code>) only runs on

{boost_skill_cli-1.0.174 → boost_skill_cli-1.0.175}/docs/roadmap.html

@@ -1019,7 +1019,7 @@ harness.</p>
       </article>
 
       <article class="cap rcard">
-        <div class="head"><span class="pill planned">Planned</span><span class="cat">Quality · Retrieval eval</span></div>
+        <div class="head"><span class="pill shipped">Shipped</span><span class="cat">Quality · Retrieval eval</span></div>
         <h3>Significance-tracked engine comparison (ranx monitor)</h3>
         <p>Tier 1b (the <code>ranx</code> paired t-test, <code>--stats</code>) only runs on
 demand today. Wire it into a scheduled CI monitor — like the

{boost_skill_cli-1.0.174 → boost_skill_cli-1.0.175}/scripts/eval_retrieval.py

@@ -18,9 +18,10 @@ the math, this asks "does the right skill actually come back for a real question
 
 Tier 1b (opt-in, offline): --stats runs a paired Student's t-test between the
 engines with `ranx`, so a metric gap is reported as statistically *significant*
-or not — a raw 0.919-vs-0.756 number can't say whether an engine is genuinely
-better or just luckier on 43 queries. Needs the [eval] extra; degrades cleanly
-if `ranx` is absent (never a core dependency).
+or not — a raw recall-vs-recall number can't say whether an engine is genuinely
+better or just luckier on the golden set. Needs the [eval] extra; degrades
+cleanly if `ranx` is absent (never a core dependency). A scheduled CI monitor
+(.github/workflows/eval-stats.yml) runs this and reports each metric's p-value.
 
 Tier 2a (LLM, opt-in, key-gated): --rerank measures the *lift* the LLM rerank
 stage (rag.rerank) adds over the raw BM25 order on the same golden set — no

boost_skill_cli-1.0.175/scripts/eval_stats_summary.py

@@ -0,0 +1,83 @@
+#!/usr/bin/env python3
+"""Render the `--stats --json` significance payload as a Markdown summary.
+
+Turns the ranx paired t-test output from ``eval_retrieval.py --stats --json``
+into a GitHub-Step-Summary table: for the best engine vs each other engine, is
+each metric's lead statistically significant (p < 0.05) or just noise. Kept
+separate from the workflow YAML so it is unit-testable (tests/unit) and so the
+significance-reading logic lives in one place.
+
+Usage:
+    python3 scripts/eval_stats_summary.py stats.json   # prints Markdown to stdout
+"""
+from __future__ import annotations
+
+import json
+import sys
+from typing import Optional
+
+ALPHA = 0.05
+HEADER = "### Tier 1b — retrieval significance (ranx paired t-test)\n"
+
+
+def _load(path: str) -> Optional[dict]:
+    try:
+        with open(path, encoding="utf-8") as fh:
+            return json.load(fh)
+    except (OSError, ValueError):
+        return None
+
+
+def render(payload: Optional[dict]) -> str:
+    """Markdown for a `--stats --json` payload (or a graceful skip note)."""
+    sig = (payload or {}).get("significance")
+    if not sig:
+        return (HEADER + "\n_skipped_ — no significance data "
+                "(the `[eval]` extra / `ranx` was unavailable, or fewer than "
+                "two engines ran).\n")
+
+    models = sig.get("model_names", [])
+    metrics = sig.get("metrics", [])
+    if len(models) < 2 or not metrics:
+        return HEADER + "\n_only one engine ran — nothing to compare._\n"
+
+    # The best engine is the last row of the comparison (engines are ordered
+    # worst→best upstream); every other engine carries the p-values of the best
+    # engine's lead over it in its `comparisons` block.
+    best = models[-1]
+    lines = [HEADER, "", "Best engine: **%s** (α = %.2f).\n" % (best, ALPHA)]
+    lines.append("| vs | " + " | ".join(metrics) + " |")
+    lines.append("|---|" + "|".join(["---"] * len(metrics)) + "|")
+
+    n_sig = 0
+    for other in models[:-1]:
+        comps = sig.get(other, {}).get("comparisons", {}).get(best, {})
+        cells = []
+        for m in metrics:
+            p = comps.get(m)
+            if p is None:
+                cells.append("—")
+            elif p < ALPHA:
+                cells.append("✅ p=%.4f" % p)
+                n_sig += 1
+            else:
+                cells.append("· p=%.4f" % p)
+        lines.append("| %s | %s |" % (other, " | ".join(cells)))
+
+    lines.append("")
+    lines.append("_✅ = %s beats that engine on the metric with p < %.2f "
+                 "(a statistically real lead, not just a higher number)._"
+                 % (best, ALPHA))
+    lines.append("\n%d of the compared metrics reach significance." % n_sig)
+    return "\n".join(lines) + "\n"
+
+
+def main(argv: Optional[list] = None) -> int:
+    args = list(sys.argv[1:] if argv is None else argv)
+    path = args[0] if args else "stats.json"
+    sys.stdout.write(render(_load(path)))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

boost_skill_cli-1.0.175/tests/unit/test_eval_stats_summary.py

@@ -0,0 +1,80 @@
+"""Unit tests for scripts/eval_stats_summary.py — the significance summary."""
+from __future__ import annotations
+
+import importlib.util
+import json
+from pathlib import Path
+
+import pytest
+
+_SCRIPT = Path(__file__).resolve().parents[2] / "scripts" / "eval_stats_summary.py"
+
+_skip = pytest.mark.skipif(not _SCRIPT.exists(), reason="script not reachable")
+
+
+def _mod():
+    spec = importlib.util.spec_from_file_location("eval_stats_summary", _SCRIPT)
+    assert spec and spec.loader
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+_PAYLOAD = {
+    "significance": {
+        "stat_test": "student",
+        "metrics": ["recall@10", "mrr"],
+        "model_names": ["catalog.search", "BM25 full-content"],
+        "catalog.search": {
+            "comparisons": {
+                "BM25 full-content": {"recall@10": 0.0047, "mrr": 0.20},
+            },
+        },
+        "BM25 full-content": {"comparisons": {}},
+    }
+}
+
+
+@_skip
+def test_significant_metric_marked_and_counted():
+    md = _mod().render(_PAYLOAD)
+    assert "Best engine: **BM25 full-content**" in md
+    assert "✅ p=0.0047" in md          # recall@10 is significant
+    assert "· p=0.2000" in md           # mrr is not
+    assert "1 of the compared metrics reach significance." in md
+
+
+@_skip
+def test_missing_significance_is_a_graceful_skip():
+    md = _mod().render({"k": 10})
+    assert "_skipped_" in md
+    assert "ranx" in md
+
+
+@_skip
+def test_none_payload_does_not_crash():
+    md = _mod().render(None)
+    assert "skipped" in md.lower()
+
+
+@_skip
+def test_single_engine_has_nothing_to_compare():
+    md = _mod().render({"significance": {
+        "metrics": ["recall@10"], "model_names": ["only-one"]}})
+    assert "nothing to compare" in md
+
+
+@_skip
+def test_main_reads_file_and_prints(tmp_path, capsys):
+    p = tmp_path / "stats.json"
+    p.write_text(json.dumps(_PAYLOAD), encoding="utf-8")
+    rc = _mod().main([str(p)])
+    assert rc == 0
+    assert "✅ p=0.0047" in capsys.readouterr().out
+
+
+@_skip
+def test_main_on_missing_file_is_graceful(capsys):
+    rc = _mod().main(["/no/such/stats.json"])
+    assert rc == 0
+    assert "skipped" in capsys.readouterr().out.lower()