boost-skill-cli 1.0.182__tar.gz → 1.0.184__tar.gz

{boost_skill_cli-1.0.182 → boost_skill_cli-1.0.184}/Makefile

@@ -56,6 +56,7 @@ lint:
 	$(PY) scripts/build_roadmap.py --check
 	$(PY) scripts/build_command_reference.py --check
 	$(PY) scripts/import_budget.py
+	$(PY) scripts/perf_gate.py
 
 # Supply-chain CVE gate: fail on a known OSV/PyPI advisory in the project's
 # dependency closure. Mirrors the pip-audit CI workflow; run before releasing.

{boost_skill_cli-1.0.182 → boost_skill_cli-1.0.184}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: boost-skill-cli
-Version: 1.0.182
+Version: 1.0.184
 Summary: boost — Homebrew for AI coding skills
 Author: Jonathan Reyes
 License: GNU GENERAL PUBLIC LICENSE

{boost_skill_cli-1.0.182 → boost_skill_cli-1.0.184}/boost_cli/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '1.0.182'
-__version_tuple__ = version_tuple = (1, 0, 182)
+__version__ = version = '1.0.184'
+__version_tuple__ = version_tuple = (1, 0, 184)
 
 __commit_id__ = commit_id = None

{boost_skill_cli-1.0.182 → boost_skill_cli-1.0.184}/boost_skill_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: boost-skill-cli
-Version: 1.0.182
+Version: 1.0.184
 Summary: boost — Homebrew for AI coding skills
 Author: Jonathan Reyes
 License: GNU GENERAL PUBLIC LICENSE

{boost_skill_cli-1.0.182 → boost_skill_cli-1.0.184}/boost_skill_cli.egg-info/SOURCES.txt

@@ -239,6 +239,7 @@ scripts/eval_retrieval.py
 scripts/eval_stats_summary.py
 scripts/import_budget.py
 scripts/mutation_gate.py
+scripts/perf_gate.py
 style/README.md
 style/boost.css
 style/boost.js
@@ -309,6 +310,7 @@ tests/unit/test_mutation_hardening.py
 tests/unit/test_nethttp.py
 tests/unit/test_output.py
 tests/unit/test_paths.py
+tests/unit/test_perf_gate.py
 tests/unit/test_pkg_confusions.py
 tests/unit/test_pkg_injection.py
 tests/unit/test_pkg_secrets.py

{boost_skill_cli-1.0.182 → boost_skill_cli-1.0.184}/docs/roadmap/items/performance-regression-gate-pytest-benchmark.md

@@ -2,15 +2,15 @@
 id: performance-regression-gate-pytest-benchmark
 board: code
 section: dx
-status: planned
+status: shipped
 category: Testing · Perf
 complexity: M
 impact: Med
 wow: 3
 note: scan & registry
 order: 6
-owner:
-pr:
+owner: loop/perfgate
+pr: 207
 title: Performance-regression gate — <code>pytest-benchmark</code>
 ---
 Benchmark the hot paths — a catalog scan over a large skills tree, a

{boost_skill_cli-1.0.182 → boost_skill_cli-1.0.184}/docs/roadmap.html

@@ -1387,7 +1387,7 @@ improvement is real (or isn't)."</p>
       </article>
 
       <article class="cap rcard">
-        <div class="head"><span class="pill planned">Planned</span><span class="cat">Testing · Perf</span></div>
+        <div class="head"><span class="pill shipped">Shipped</span><span class="cat">Testing · Perf</span></div>
         <h3>Performance-regression gate — <code>pytest-benchmark</code></h3>
         <p>Benchmark the hot paths — a catalog scan over a large skills tree, a
            registry load — and fail the build when a change regresses them past a

boost_skill_cli-1.0.184/scripts/perf_gate.py

@@ -0,0 +1,149 @@
+#!/usr/bin/env python3
+"""Performance-regression gate: complexity scaling, not wall-clock baselines.
+
+A saved-baseline benchmark compare flakes on shared CI runners — machine speed
+varies run to run, so absolute thresholds either false-alarm or sandbag. What
+a gate CAN assert robustly is *how cost grows with input size*: time the hot
+paths at two catalog sizes on the SAME machine in the SAME process, and fail
+when the big/small ratio blows past the size ratio times a slack factor. An
+accidental O(n²) (size ratio 8 → time ratio ~64×) is caught decisively; noise
+and slow runners cancel out of the ratio. One generous absolute backstop
+catches only catch-fire regressions.
+
+Paths gated (the ones `boost search`/`tap` lean on):
+  * catalog.scan_dir  — walking + classifying a tap tree
+  * rag.build         — BM25 indexing of the scanned entries
+  * rag.retrieve      — query over the built index
+
+Usage:
+  PYTHONPATH=. python3 scripts/perf_gate.py          # run the gate
+  PYTHONPATH=. python3 scripts/perf_gate.py -v       # per-path timings
+
+Deliberately NOT part of `make check`'s five-gate contract (v1 blast-radius);
+CI runs it in the lint job. The opt-in micro-benchmarks (tests/perf, `make
+bench`) remain the profiling tool; this is the regression tripwire.
+"""
+from __future__ import annotations
+
+import argparse
+import os
+import sys
+import tempfile
+import time
+from pathlib import Path
+
+# Run from a source checkout without an install (same shim as the eval gate).
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+SMALL, BIG = 64, 512          # catalog sizes; 8× input
+SLACK = 3.0                   # fail when time-ratio > size-ratio × SLACK (24×)
+ABS_BACKSTOP_S = 30.0         # any single big-size op slower than this fails
+MIN_SAMPLE_S = 0.05           # repeat an op until a sample is ≥ 50ms (jitter)
+BEST_OF = 3
+
+
+def _mk_catalog(root: Path, n: int) -> None:
+    """Materialize a synthetic tap tree with n skills + n rules + n workflows.
+
+    Content scales with the tree so indexing has real text to chew on, and
+    names are unique per size so nothing is accidentally shared or cached.
+    """
+    lorem = ["retrieval", "augmentation", "index", "tokens", "ranking", "corpus", "recall", "precision", "embedding", "catalog", "skill", "agent", "workflow", "rule"]
+    for i in range(n):
+        d = root / ("skill-%04d" % i)
+        d.mkdir(parents=True)
+        words = " ".join(lorem[(i + j) % len(lorem)] for j in range(60))
+        (d / "SKILL.md").write_text(
+            "---\nname: skill-%04d\ndescription: synthetic skill %d for the "
+            "perf gate\n---\n\n# skill-%04d\n\n%s\n" % (i, i, i, words),
+            encoding="utf-8")
+        (root / "rules" / ("rule-%04d.mdc" % i)).parent.mkdir(exist_ok=True)
+        (root / "rules" / ("rule-%04d.mdc" % i)).write_text(
+            "---\ndescription: synthetic rule %d\n---\nAlways %s.\n"
+            % (i, " ".join(lorem[i % len(lorem)] for _ in range(20))),
+            encoding="utf-8")
+        (root / "commands" / ("wf-%04d.md" % i)).parent.mkdir(exist_ok=True)
+        (root / "commands" / ("wf-%04d.md" % i)).write_text(
+            "# wf-%04d\n\nRun the synthetic workflow %d: %s\n"
+            % (i, i, words[:120]), encoding="utf-8")
+
+
+def _time(fn) -> float:
+    """Best-of-BEST_OF timing; each sample loops fn until ≥ MIN_SAMPLE_S."""
+    best = float("inf")
+    for _ in range(BEST_OF):
+        reps = 0
+        t0 = time.perf_counter()
+        while True:
+            fn()
+            reps += 1
+            dt = time.perf_counter() - t0
+            if dt >= MIN_SAMPLE_S:
+                break
+        best = min(best, dt / reps)
+    return best
+
+
+def _measure(size: int, verbose: bool) -> dict:
+    """Time the three hot paths against a fresh synthetic catalog of `size`."""
+    from boost_cli.core import catalog, rag
+
+    with tempfile.TemporaryDirectory(prefix="perfgate-") as tmp:
+        tap = Path(tmp) / "tap"
+        _mk_catalog(tap, size)
+        os.environ["BOOST_HOME"] = str(Path(tmp) / "home")
+        os.environ["HOME"] = tmp
+
+        out = {"scan": _time(lambda: catalog.scan_dir(tap, "perf"))}
+        entries = catalog.scan_dir(tap, "perf")
+        if len(entries) != 3 * size:  # a generator bug would invalidate ratios
+            raise SystemExit("perf-gate: generator drift: %d entries for n=%d"
+                             % (len(entries), size))
+
+        # build is once-per-corpus in real life — time single builds (forced).
+        best = float("inf")
+        for _ in range(BEST_OF):
+            t0 = time.perf_counter()
+            rag.build(entries, force=True)
+            best = min(best, time.perf_counter() - t0)
+        out["build"] = best
+
+        out["retrieve"] = _time(
+            lambda: rag.retrieve("ranking corpus recall skill", k=20))
+        if verbose:
+            for k, v in out.items():
+                print("  n=%-4d %-9s %8.1f ms" % (size, k, v * 1000))
+        return out
+
+
+def main(argv=None) -> int:
+    ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    ap.add_argument("-v", "--verbose", action="store_true")
+    args = ap.parse_args(argv)
+
+    small = _measure(SMALL, args.verbose)
+    big = _measure(BIG, args.verbose)
+
+    size_ratio = BIG / SMALL
+    limit = size_ratio * SLACK
+    failed = []
+    for path in ("scan", "build", "retrieve"):
+        ratio = big[path] / small[path] if small[path] > 0 else float("inf")
+        line = "%-9s %7.1fms -> %8.1fms  ratio %5.1fx (limit %.0fx)" % (
+            path, small[path] * 1000, big[path] * 1000, ratio, limit)
+        ok = ratio <= limit and big[path] <= ABS_BACKSTOP_S
+        print(("  PASS " if ok else "  FAIL ") + line)
+        if not ok:
+            failed.append(path)
+
+    if failed:
+        print("perf-gate: FAILED — superlinear scaling (or >%ds backstop) on: %s"
+              % (int(ABS_BACKSTOP_S), ", ".join(failed)), file=sys.stderr)
+        return 1
+    print("perf-gate: OK — all paths scale within %.0fx of linear at %dx input"
+          % (SLACK, int(size_ratio)))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

boost_skill_cli-1.0.184/tests/unit/test_perf_gate.py

@@ -0,0 +1,64 @@
+"""Unit tests for scripts/perf_gate.py — the scaling-ratio decision logic."""
+from __future__ import annotations
+
+import importlib.util
+from pathlib import Path
+
+import pytest
+
+_SCRIPT = Path(__file__).resolve().parents[2] / "scripts" / "perf_gate.py"
+
+_skip = pytest.mark.skipif(not _SCRIPT.exists(), reason="script not reachable")
+
+
+def _mod():
+    spec = importlib.util.spec_from_file_location("perf_gate", _SCRIPT)
+    assert spec and spec.loader
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+@_skip
+def test_linear_scaling_passes(monkeypatch, capsys):
+    pg = _mod()
+    fake = {64: {"scan": .01, "build": .01, "retrieve": .001},
+            512: {"scan": .08, "build": .08, "retrieve": .008}}   # exactly 8x
+    monkeypatch.setattr(pg, "_measure", lambda n, v: fake[n])
+    assert pg.main([]) == 0
+    assert "perf-gate: OK" in capsys.readouterr().out
+
+
+@_skip
+def test_quadratic_scaling_fails(monkeypatch, capsys):
+    pg = _mod()
+    fake = {64: {"scan": .01, "build": .01, "retrieve": .001},
+            512: {"scan": .64, "build": .08, "retrieve": .008}}   # scan at 64x
+    monkeypatch.setattr(pg, "_measure", lambda n, v: fake[n])
+    assert pg.main([]) == 1
+    out = capsys.readouterr()
+    assert "FAIL scan" in out.out
+    assert "superlinear" in out.err
+
+
+@_skip
+def test_absolute_backstop_fails_even_at_linear_ratio(monkeypatch, capsys):
+    pg = _mod()
+    fake = {64: {"scan": 5.0, "build": .01, "retrieve": .001},
+            512: {"scan": 40.0, "build": .08, "retrieve": .008}}  # 8x but 40s
+    monkeypatch.setattr(pg, "_measure", lambda n, v: fake[n])
+    assert pg.main([]) == 1
+    assert "FAIL scan" in capsys.readouterr().out
+
+
+@_skip
+def test_synthetic_catalog_has_expected_entry_count(tmp_path):
+    pg = _mod()
+    pg._mk_catalog(tmp_path, 5)
+    import boost_cli.core.catalog as catalog
+    entries = catalog.scan_dir(tmp_path, "t")
+    kinds = sorted(e.get("kind", "skill") for e in entries)
+    assert len(entries) == 15
+    assert kinds.count("skill") == 5
+    assert kinds.count("rule") == 5
+    assert kinds.count("workflow") == 5