boost-skill-cli 1.0.441__tar.gz → 1.0.443__tar.gz

{boost_skill_cli-1.0.441/boost_skill_cli.egg-info → boost_skill_cli-1.0.443}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: boost-skill-cli
-Version: 1.0.441
+Version: 1.0.443
 Summary: boost — Homebrew for AI coding skills
 Author: Jonathan Reyes
 License: GNU GENERAL PUBLIC LICENSE

{boost_skill_cli-1.0.441 → boost_skill_cli-1.0.443}/boost_cli/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '1.0.441'
-__version_tuple__ = version_tuple = (1, 0, 441)
+__version__ = version = '1.0.443'
+__version_tuple__ = version_tuple = (1, 0, 443)
 
 __commit_id__ = commit_id = None

{boost_skill_cli-1.0.441 → boost_skill_cli-1.0.443/boost_skill_cli.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: boost-skill-cli
-Version: 1.0.441
+Version: 1.0.443
 Summary: boost — Homebrew for AI coding skills
 Author: Jonathan Reyes
 License: GNU GENERAL PUBLIC LICENSE

{boost_skill_cli-1.0.441 → boost_skill_cli-1.0.443}/boost_skill_cli.egg-info/SOURCES.txt

@@ -241,6 +241,7 @@ docs/roadmap/items/eval-corpus-is-one-strangers-repo.md
 docs/roadmap/items/eval-corpus-pins-have-no-refresh-path.md
 docs/roadmap/items/eval-corpus-was-not-actually-pinned.md
 docs/roadmap/items/eval-deduped-ranked-lists-by-name.md
+docs/roadmap/items/eval-stats-could-never-publish.md
 docs/roadmap/items/expanded-card-bodies-overflow.md
 docs/roadmap/items/extension-free-tests-core-dense-py.md
 docs/roadmap/items/extract-mcp-http-servers-out-of-configuration-py.md
@@ -640,6 +641,7 @@ tests/unit/test_post_deploy_smoke.py
 tests/unit/test_project_mcp_sidecar.py
 tests/unit/test_projectlock.py
 tests/unit/test_provenance.py
+tests/unit/test_publish_golden_dataset.py
 tests/unit/test_rag.py
 tests/unit/test_rag_fusion.py
 tests/unit/test_registries_fresh.py
@@ -670,6 +672,7 @@ tests/unit/test_util.py
 tests/unit/test_version.py
 tests/unit/test_workflow_concurrency.py
 tests/unit/test_workflow_concurrency_groups.py
+tests/unit/test_workflow_push_has_credentials.py
 tests/unit/test_workflow_timeouts.py
 tests/unit/test_workflows.py
 tests/visual/a11y_check.mjs

boost_skill_cli-1.0.443/docs/roadmap/items/eval-stats-could-never-publish.md

@@ -0,0 +1,45 @@
+---
+id: eval-stats-could-never-publish
+board: code
+section: pipeline
+status: shipped
+category: Build · Bug
+complexity: S
+impact: Med
+wow: 4
+note: the published-metrics file has ONE commit in its whole history — the one that created it
+order: 109
+owner: fix/eval-stats-push-auth
+pr:
+title: The published metrics could never be published
+---
+<code>eval-stats.yml</code> regenerates <code>docs/eval-latest.json</code> — the payload the docs site
+reads — commits it, and pushes to <code>main</code> on a schedule. The proof that it never landed is
+in the file's own history: <b>one commit, ever</b>, the one that created it in
+<code>#421</code>.
+
+<b>The cause is a pair of individually-correct lines.</b> The checkout sets
+<code>persist-credentials: false</code>, which is right and is what zizmor's
+<code>artipacked</code> rule asks for. Ninety lines later the job runs
+<code>git push origin HEAD:main</code>. With no persisted credentials the remote carries no auth, so
+that push cannot work — and nothing in either line looks wrong on its own. The defect only exists
+in the distance between them.
+
+<b>It reported success while doing nothing, which is why it survived.</b> The step exits 0 early
+when there is nothing to commit, so a run that found no metric drift and a run that could not
+publish are the same green tick. Of the two scheduled runs, one passed that way and one failed —
+and the failure was invisible for the same reason <code>fuzz</code> and <code>shards</code> were:
+until <code>#508</code>, <code>ci-failure-issue.yml</code> watched 2 of 26 workflows, and a cron
+job's failure is a red square on a page nobody opens.
+
+<b>The fix was already in the repo.</b> <code>ci.yml</code> pushes its badges branch to an explicit
+<code>x-access-token</code> URL — keeping the safe checkout default <i>and</i> being able to push.
+The same three lines applied here. Nothing about the problem was novel; what was missing was anything
+that compared the two workflows.
+
+So the guard does that comparison. <code>tests/unit/test_workflow_push_has_credentials.py</code>
+fails the build when a workflow checks out without persisting credentials and then pushes without
+supplying any. It folds shell line-continuations before matching, because a push carrying a token in
+its URL is long enough to wrap — matching per physical line would have failed the correct fix and
+passed the broken one, which is the failure mode a guard can least afford. Both directions were
+verified: green on the fix, and red again when the bare push is restored.

{boost_skill_cli-1.0.441 → boost_skill_cli-1.0.443}/docs/roadmap.html

@@ -133,7 +133,7 @@
       <div class="stat"><b>16</b><span>Shipped</span></div>
       <div class="stat"><b>1</b><span>Next up</span></div>
       <div class="stat"><b>7</b><span>Planned</span></div>
-      <div class="stat"><b>239</b><span>Loop finds</span></div>
+      <div class="stat"><b>240</b><span>Loop finds</span></div>
       <!-- /ROADMAP:stats -->
     </div>
   </section>
@@ -6111,6 +6111,48 @@ vacuously if the trigger detection silently stops matching.</p></details>
           <span class="m">two PRs deadlocked each other for hours — re-running the job just moved the cancellation to the other one</span>
         </div>
       </article>
+
+      <article class="cap rcard" id="eval-stats-could-never-publish">
+        <div class="head"><span class="pill shipped">Shipped</span><span class="cat">Build · Bug</span></div>
+        <h3>The published metrics could never be published</h3>
+        <details class="cardbody"><summary>Write-up</summary>
+        <p><code>eval-stats.yml</code> regenerates <code>docs/eval-latest.json</code> — the payload the docs site
+reads — commits it, and pushes to <code>main</code> on a schedule. The proof that it never landed is
+in the file's own history: <b>one commit, ever</b>, the one that created it in
+<code>#421</code>.
+
+<b>The cause is a pair of individually-correct lines.</b> The checkout sets
+<code>persist-credentials: false</code>, which is right and is what zizmor's
+<code>artipacked</code> rule asks for. Ninety lines later the job runs
+<code>git push origin HEAD:main</code>. With no persisted credentials the remote carries no auth, so
+that push cannot work — and nothing in either line looks wrong on its own. The defect only exists
+in the distance between them.
+
+<b>It reported success while doing nothing, which is why it survived.</b> The step exits 0 early
+when there is nothing to commit, so a run that found no metric drift and a run that could not
+publish are the same green tick. Of the two scheduled runs, one passed that way and one failed —
+and the failure was invisible for the same reason <code>fuzz</code> and <code>shards</code> were:
+until <code>#508</code>, <code>ci-failure-issue.yml</code> watched 2 of 26 workflows, and a cron
+job's failure is a red square on a page nobody opens.
+
+<b>The fix was already in the repo.</b> <code>ci.yml</code> pushes its badges branch to an explicit
+<code>x-access-token</code> URL — keeping the safe checkout default <i>and</i> being able to push.
+The same three lines applied here. Nothing about the problem was novel; what was missing was anything
+that compared the two workflows.
+
+So the guard does that comparison. <code>tests/unit/test_workflow_push_has_credentials.py</code>
+fails the build when a workflow checks out without persisting credentials and then pushes without
+supplying any. It folds shell line-continuations before matching, because a push carrying a token in
+its URL is long enough to wrap — matching per physical line would have failed the correct fix and
+passed the broken one, which is the failure mode a guard can least afford. Both directions were
+verified: green on the fix, and red again when the bare push is restored.</p></details>
+        <div class="meta">
+          <span class="m">Complexity <b>S</b></span>
+          <span class="m">Impact <b>Med</b></span>
+          <span class="m">Wow <b class="wow">★★★★</b></span>
+          <span class="m">the published-metrics file has ONE commit in its whole history — the one that created it</span>
+        </div>
+      </article>
       <!-- /ROADMAP:cards -->
 
     </div>

boost_skill_cli-1.0.443/tests/unit/test_publish_golden_dataset.py

@@ -0,0 +1,226 @@
+"""Unit tests: the LangSmith golden-set publisher, which had none.
+
+``evals/publish_golden_dataset.py`` is the "observe" leg of boost's LangChain
+integration — it mirrors ``tests/eval/golden.jsonl`` into a LangSmith dataset so
+online evals grade against the same ground truth the offline Tier-1 gate floors.
+It is deliberately outside the required gate (a required check that depends on a
+SaaS account fails when someone else's billing lapses), and the cost of that is
+that **nothing exercised it at all**: no test imported it, so every path
+including the argument parsing was unverified.
+
+It cannot be tested against the real service, and should not be. What *is*
+testable is everything the network never sees, which is where the bugs a reader
+would actually hit live:
+
+* the two opt-in guards return cleanly rather than tracebacking, which is the
+  documented contract for a surface that must degrade when unconfigured;
+* the golden file parses the way the offline gate parses it — comments and
+  blanks dropped — because a publisher that silently disagrees with the gate
+  about which rows count would publish a dataset that grades differently while
+  claiming to be the same ground truth;
+* every row becomes exactly one example with the query as input and the
+  judgments as output.
+
+The client is a stub. That is not a shortcut around the interesting part: the
+interesting part is the mapping from a golden row to an example, and a stub is
+what makes it observable.
+"""
+from __future__ import annotations
+
+import importlib.util
+import json
+from pathlib import Path
+
+import pytest
+
+ROOT = Path(__file__).resolve().parents[2]
+SCRIPT = ROOT / "evals" / "publish_golden_dataset.py"
+GOLDEN = ROOT / "tests" / "eval" / "golden.jsonl"
+
+pytestmark = pytest.mark.skipif(
+    not SCRIPT.exists(), reason="evals/ not reachable (e.g. mutation sandbox)")
+
+
+def load_publisher():
+    """Import the script by path, the way this repo tests scripts/."""
+    spec = importlib.util.spec_from_file_location("publish_golden_dataset",
+                                                  SCRIPT)
+    mod = importlib.util.module_from_spec(spec)
+    assert spec.loader is not None
+    spec.loader.exec_module(mod)
+    return mod
+
+
+MOD = load_publisher()
+
+
+class TestLoadGolden:
+    """The parse must agree with the gate's, or the two grade different sets."""
+
+    def test_comments_and_blanks_are_dropped(self, tmp_path):
+        path = tmp_path / "g.jsonl"
+        path.write_text(
+            '# a comment\n\n{"query": "q1", "relevant": ["a"]}\n'
+            '   \n{"query": "q2", "relevant": ["b"]}\n', encoding="utf-8")
+        assert [r["query"] for r in MOD.load_golden(path)] == ["q1", "q2"]
+
+    def test_the_real_golden_set_parses(self):
+        rows = MOD.load_golden(GOLDEN)
+        assert rows, "the shipped golden set parsed as empty"
+        assert all("query" in r for r in rows)
+
+    def test_it_agrees_with_the_gate_on_how_many_rows_there_are(self):
+        # THE POINT of testing the parser at all. The offline gate and this
+        # publisher both read golden.jsonl; if they disagree about which lines
+        # count, the LangSmith dataset silently grades a different set while
+        # claiming to be the same ground truth.
+        expected = sum(
+            1 for line in GOLDEN.read_text(encoding="utf-8").splitlines()
+            if line.strip() and not line.strip().startswith("#"))
+        assert len(MOD.load_golden(GOLDEN)) == expected
+
+
+class TestItDegradesWhenUnconfigured:
+    """Both guards are documented as "message, not traceback". Pin that."""
+
+    def test_a_missing_key_returns_nonzero_without_raising(self, monkeypatch,
+                                                           capsys):
+        monkeypatch.delenv("LANGSMITH_API_KEY", raising=False)
+        assert MOD.main(["--golden", str(GOLDEN)]) == 1
+        assert "LANGSMITH_API_KEY" in capsys.readouterr().err
+
+    def test_a_missing_package_returns_nonzero_without_raising(self, monkeypatch,
+                                                              capsys):
+        monkeypatch.setenv("LANGSMITH_API_KEY", "not-a-real-key")
+        monkeypatch.setitem(__import__("sys").modules, "langsmith", None)
+        assert MOD.main(["--golden", str(GOLDEN)]) == 1
+        assert "langsmith" in capsys.readouterr().err
+
+    def test_the_key_check_happens_before_any_client_is_built(self, monkeypatch):
+        # Ordering, not politeness: reaching a Client() constructor without a
+        # key is how an opt-in surface turns into a traceback on a machine that
+        # simply never opted in.
+        monkeypatch.delenv("LANGSMITH_API_KEY", raising=False)
+        built = []
+        monkeypatch.setitem(__import__("sys").modules, "langsmith",
+                            _fake_langsmith(built))
+        MOD.main(["--golden", str(GOLDEN)])
+        assert built == []
+
+
+class _FakeDataset:
+    def __init__(self, ident="ds-1"):
+        self.id = ident
+
+
+class _FakeClient:
+    """Enough of langsmith.Client to observe what would be sent."""
+
+    def __init__(self, existing=False, calls=None):
+        self.calls = calls if calls is not None else []
+        self._existing = existing
+        self.created_examples = []
+        self.deleted = []
+
+    def has_dataset(self, dataset_name):
+        self.calls.append(("has_dataset", dataset_name))
+        return self._existing
+
+    def read_dataset(self, dataset_name):
+        self.calls.append(("read_dataset", dataset_name))
+        return _FakeDataset()
+
+    def create_dataset(self, dataset_name, description=""):
+        self.calls.append(("create_dataset", dataset_name))
+        return _FakeDataset()
+
+    def list_examples(self, dataset_id):
+        self.calls.append(("list_examples", dataset_id))
+        return [_FakeDataset("ex-1"), _FakeDataset("ex-2")]
+
+    def delete_example(self, example_id):
+        self.deleted.append(example_id)
+
+    def create_examples(self, dataset_id, examples):
+        self.calls.append(("create_examples", dataset_id))
+        self.created_examples = examples
+
+
+def _fake_langsmith(built, existing=False, box=None):
+    """A stand-in ``langsmith`` module whose Client records what it was asked."""
+    import types
+    mod = types.ModuleType("langsmith")
+
+    def _client(*a, **k):
+        client = _FakeClient(existing=existing)
+        built.append(client)
+        if box is not None:
+            box.append(client)
+        return client
+
+    mod.Client = _client
+    return mod
+
+
+def _run(monkeypatch, tmp_path, rows, existing=False):
+    """Publish ``rows`` through a stub client; return that client."""
+    path = tmp_path / "g.jsonl"
+    path.write_text("\n".join(json.dumps(r) for r in rows), encoding="utf-8")
+    monkeypatch.setenv("LANGSMITH_API_KEY", "not-a-real-key")
+    built: list = []
+    monkeypatch.setitem(__import__("sys").modules, "langsmith",
+                        _fake_langsmith(built, existing=existing))
+    rc = MOD.main(["--golden", str(path), "--name", "test-set"])
+    assert rc == 0
+    assert built, "no client was constructed"
+    return built[0]
+
+
+ROWS = [{"query": "how do I review code", "relevant": ["code-review"],
+         "kind": "skill", "note": "n1"},
+        {"query": "commit conventions", "relevant": ["conventional-commits"],
+         "kind": "rule", "note": ""}]
+
+
+class TestEveryRowBecomesAnExample:
+    def test_one_example_per_row(self, monkeypatch, tmp_path):
+        client = _run(monkeypatch, tmp_path, ROWS)
+        assert len(client.created_examples) == len(ROWS)
+
+    def test_the_query_is_the_input(self, monkeypatch, tmp_path):
+        client = _run(monkeypatch, tmp_path, ROWS)
+        assert [e["inputs"]["query"] for e in client.created_examples] == \
+            [r["query"] for r in ROWS]
+
+    def test_the_judgments_are_the_output(self, monkeypatch, tmp_path):
+        # This is the ground truth an online eval grades against; losing it
+        # would leave a dataset of questions with no answers, which still
+        # publishes cleanly.
+        client = _run(monkeypatch, tmp_path, ROWS)
+        assert client.created_examples[0]["outputs"]["relevant"] == \
+            ["code-review"]
+        assert client.created_examples[0]["outputs"]["kind"] == "skill"
+
+    def test_a_row_missing_optional_fields_still_publishes(self, monkeypatch,
+                                                          tmp_path):
+        # golden.jsonl's optional keys are optional in the gate too.
+        client = _run(monkeypatch, tmp_path, [{"query": "bare"}])
+        example = client.created_examples[0]
+        assert example["outputs"]["relevant"] == []
+        assert example["metadata"]["note"] == ""
+
+
+class TestItMirrorsRatherThanAccretes:
+    """Re-running must leave the dataset equal to the file, not appended to."""
+
+    def test_a_new_dataset_is_created_when_absent(self, monkeypatch, tmp_path):
+        client = _run(monkeypatch, tmp_path, ROWS, existing=False)
+        assert ("create_dataset", "test-set") in client.calls
+
+    def test_existing_examples_are_deleted_before_the_new_ones_land(
+            self, monkeypatch, tmp_path):
+        # Without this, publishing twice doubles every example and the dataset
+        # quietly stops matching the file it claims to mirror.
+        client = _run(monkeypatch, tmp_path, ROWS, existing=True)
+        assert client.deleted, "no existing examples were removed"
+        assert ("create_dataset", "test-set") not in client.calls

boost_skill_cli-1.0.443/tests/unit/test_workflow_push_has_credentials.py

@@ -0,0 +1,100 @@
+"""Unit tests: a workflow that pushes has credentials to push with.
+
+``actions/checkout`` with ``persist-credentials: false`` leaves no auth in the
+remote — which is the right default, and what zizmor's ``artipacked`` rule asks
+for. A later bare ``git push origin`` in the same job therefore cannot work.
+
+``eval-stats.yml`` had exactly that pair. It regenerates
+``docs/eval-latest.json`` — the payload the docs site reads — commits it, and
+pushes. The proof that it never landed is in the file's own history: **one
+commit, ever**, the one that created it. Every scheduled refresh since has
+either found nothing to commit or died on the push.
+
+It is the same silence as ``shards`` and ``fuzz``: a workflow that looks like a
+control and produces nothing, on a cron nobody watches. The run even reports
+*success* when there is nothing to commit, so the conclusion column cannot
+distinguish "no drift" from "cannot publish".
+
+``ci.yml`` already had the fix in the repo — it pushes its badges branch with an
+explicit ``x-access-token`` URL, keeping the safe checkout default *and* being
+able to push. This test pins the pairing so the two halves cannot drift apart
+again in either direction.
+"""
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+import pytest
+
+WORKFLOWS = Path(__file__).resolve().parents[2] / ".github" / "workflows"
+
+pytestmark = pytest.mark.skipif(
+    not WORKFLOWS.exists(),
+    reason=".github/workflows not reachable (e.g. mutation sandbox)")
+
+#: A push that carries its own credentials, rather than relying on the ones the
+#: checkout deliberately did not persist.
+_AUTHED_PUSH = re.compile(r"git push[^\n]*(x-access-token|\$\{\{\s*secrets\.|"
+                          r"\$\{\{\s*github\.token|GITHUB_TOKEN|GH_TOKEN)")
+_ANY_PUSH = re.compile(r"^\s*git push\b", re.M)
+
+
+def join_continuations(text: str) -> str:
+    """Fold shell line-continuations, so one command is one line.
+
+    A push long enough to need credentials in its URL is a push long enough to
+    be wrapped, and matching per physical line would read the token on the next
+    line as absent — failing a workflow that is correct. Folding first is what
+    the shell itself does.
+    """
+    return re.sub(r"\\\n\s*", " ", text)
+
+
+def pushing_workflows() -> dict[str, str]:
+    """``{file name: folded text}`` for every workflow containing a git push."""
+    out = {}
+    for path in sorted(WORKFLOWS.glob("*.yml")):
+        text = join_continuations(path.read_text(encoding="utf-8"))
+        if _ANY_PUSH.search(text):
+            out[path.name] = text
+    return out
+
+
+class TestTheGuardCanActuallySee:
+    def test_at_least_one_workflow_pushes(self):
+        # Otherwise every assertion below is vacuous.
+        assert pushing_workflows(), "no workflow matched a git push"
+
+    def test_an_authed_push_is_recognised(self):
+        line = ('git push -qf "https://x-access-token:${{ github.token }}'
+                '@github.com/${{ github.repository }}" badges:badges')
+        assert _AUTHED_PUSH.search(line)
+
+    def test_a_bare_push_is_not_recognised_as_authed(self):
+        assert not _AUTHED_PUSH.search("          git push origin HEAD:main")
+
+    def test_a_wrapped_authed_push_is_recognised(self):
+        # The shape the fix actually takes: a token URL is too long for one
+        # line, so without folding the continuation this guard would fail a
+        # workflow that is correct.
+        wrapped = ('          git push -q \\\n'
+                   '            "https://x-access-token:${{ github.token }}'
+                   '@github.com/${{ github.repository }}" \\\n'
+                   '            HEAD:main\n')
+        assert _AUTHED_PUSH.search(join_continuations(wrapped))
+
+
+@pytest.mark.parametrize("name", sorted(pushing_workflows()))
+def test_a_push_after_an_unpersisted_checkout_carries_a_token(name):
+    text = pushing_workflows()[name]
+    if "persist-credentials: false" not in text:
+        pytest.skip("%s keeps the checkout credentials" % name)
+    bare = [line.strip() for line in text.splitlines()
+            if _ANY_PUSH.match(line) and not _AUTHED_PUSH.search(line)]
+    assert not bare, (
+        "%s checks out with persist-credentials: false — so the remote has no "
+        "auth — and then pushes without supplying any: %s. eval-stats.yml did "
+        "exactly this and never once refreshed docs/eval-latest.json; the "
+        "file has one commit in its whole history. ci.yml shows the fix: push "
+        "to an explicit x-access-token URL." % (name, bare))