boost-skill-cli 1.2.9__tar.gz → 1.2.10__tar.gz

{boost_skill_cli-1.2.9 → boost_skill_cli-1.2.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: boost-skill-cli
-Version: 1.2.9
+Version: 1.2.10
 Summary: boost — Homebrew for AI coding skills
 Author: Jonathan Reyes
 License: GNU GENERAL PUBLIC LICENSE

{boost_skill_cli-1.2.9 → boost_skill_cli-1.2.10}/boost_cli/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '1.2.9'
-__version_tuple__ = version_tuple = (1, 2, 9)
+__version__ = version = '1.2.10'
+__version_tuple__ = version_tuple = (1, 2, 10)
 
 __commit_id__ = commit_id = None

{boost_skill_cli-1.2.9 → boost_skill_cli-1.2.10}/boost_skill_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: boost-skill-cli
-Version: 1.2.9
+Version: 1.2.10
 Summary: boost — Homebrew for AI coding skills
 Author: Jonathan Reyes
 License: GNU GENERAL PUBLIC LICENSE

{boost_skill_cli-1.2.9 → boost_skill_cli-1.2.10}/boost_skill_cli.egg-info/SOURCES.txt

@@ -412,6 +412,7 @@ docs/roadmap/items/sync-reported-success-for-a-link-it-refused.md
 docs/roadmap/items/tap-signing-and-provenance-sigstore-minisign.md
 docs/roadmap/items/taps-check-out-freight-they-never-index.md
 docs/roadmap/items/the-catalogue-advertised-repos-that-no-longer-exist.md
+docs/roadmap/items/the-scale-corpus-refreshed-the-wrong-rows.md
 docs/roadmap/items/the-shard-job-that-could-not-finish.md
 docs/roadmap/items/theme-asset-linting-stylelint-eslint.md
 docs/roadmap/items/tool-call-eval-tier.md

boost_skill_cli-1.2.10/docs/roadmap/items/the-scale-corpus-refreshed-the-wrong-rows.md

@@ -0,0 +1,53 @@
+---
+id: the-scale-corpus-refreshed-the-wrong-rows
+board: code
+section: internals
+status: shipped
+category: CI · Evaluation
+complexity: S
+impact: High
+wow: 5
+note: 185 rows, 20 pinned — the monthly job moved exactly the 20 it did not own and pinned none of the 165 it existed to pin
+order: 121
+owner: loop/scale-refresh-freezes-required
+pr: 537
+title: the scheduled re-pin that refreshed the twenty rows it must not touch, and none of the hundred and sixty-five it existed to pin
+---
+<b>The Tier 1b scale corpus is the required corpus plus distractors</b> &mdash; 20 rows copied
+verbatim out of <code>taps.txt</code>, pins and counts included, so both tiers start from the same
+trees, plus 165 curated registries that make it the size of a real install. The golden floors were
+calibrated against those 20 trees. A scale tier measuring a <em>different</em> snapshot of the same
+repos reports a difference that is not scale.
+
+<b>The monthly job did precisely the inverse of its job.</b>
+<code>eval_corpus.py --refresh</code> walked every row of whatever file it was handed, so it moved
+the 20 required pins &mdash; which <code>build_scale_corpus.py</code> owns and copies verbatim
+&mdash; and, because <code>relock_text</code> skipped any row with fewer than two fields, pinned
+<b>zero</b> of the 165 bare distractors. The tier that exists to measure a pinned 20,000-entry
+corpus was floating on upstream HEAD for <b>89% of its rows</b>.
+
+<b>It also could never merge.</b> The same workflow runs
+<code>build_scale_corpus.py --check</code> one step <em>before</em> the refresh; the refresh then
+broke exactly that check. Every PR it opened was unmergeable by construction, and the diff read as
+ordinary pin movement &mdash; 20 SHAs and 20 counts, which is what a healthy re-pin looks like.
+PR #536 is the specimen.
+
+<b>Two mechanisms, both about a fallback being consulted too early.</b> The pins moved because
+nothing said the required rows belong to another file. The distractors stayed bare because
+<code>relock_text</code> gated on <em>&ldquo;does this row already have a pin&rdquo;</em> when the
+durable rule is <em>&ldquo;never write a count without a SHA&rdquo;</em> &mdash;
+<code>--relock</code> re-measures the same tree and has no SHA to offer, but <code>--refresh</code>
+has just resolved upstream HEAD and can close the row. A third, quieter symptom fell out of the
+first: the column width is the longest name being rewritten, and the distractor names are longer
+than anything in <code>taps.txt</code>, so every required row was re-columned even where its pin had
+not changed.
+
+<b>Freezing is by membership, not identity.</b> <code>--refresh</code> on <code>taps.txt</code>
+itself is exactly the job that <em>may</em> move those pins, so it freezes nothing. Freezing on
+&ldquo;is this a required repo&rdquo; would have made the required corpus permanently unrefreshable
+&mdash; the same bug pointed the other way, and far quieter.
+
+<b>Verified against the real file, not argued.</b> Simulating a refresh that resolved a new SHA for
+all 185 rows: the 20 required rows come back byte-identical, all 185 end up pinned, and
+<code>--check</code> passes &mdash; so the PR is mergeable. The workflow now runs that check
+<b>after</b> the refresh as well as before, which is the guard that was missing.

{boost_skill_cli-1.2.9 → boost_skill_cli-1.2.10}/docs/roadmap.html

@@ -133,7 +133,7 @@
       <div class="stat"><b>16</b><span>Shipped</span></div>
       <div class="stat"><b>1</b><span>Next up</span></div>
       <div class="stat"><b>7</b><span>Planned</span></div>
-      <div class="stat"><b>258</b><span>Loop finds</span></div>
+      <div class="stat"><b>259</b><span>Loop finds</span></div>
       <!-- /ROADMAP:stats -->
     </div>
   </section>
@@ -4255,6 +4255,56 @@ with the numbers.</p></details>
           <span class="m">5h30m cancelled at the ceiling, then 2h07m green — the same job, with duplicate embeddings collapsed</span>
         </div>
       </article>
+
+      <article class="cap rcard" id="the-scale-corpus-refreshed-the-wrong-rows">
+        <div class="head"><span class="pill shipped">Shipped</span><span class="cat">CI · Evaluation</span></div>
+        <h3>the scheduled re-pin that refreshed the twenty rows it must not touch, and none of the hundred and sixty-five it existed to pin</h3>
+        <details class="cardbody"><summary>Write-up</summary>
+        <p><b>The Tier 1b scale corpus is the required corpus plus distractors</b> &mdash; 20 rows copied
+verbatim out of <code>taps.txt</code>, pins and counts included, so both tiers start from the same
+trees, plus 165 curated registries that make it the size of a real install. The golden floors were
+calibrated against those 20 trees. A scale tier measuring a <em>different</em> snapshot of the same
+repos reports a difference that is not scale.
+
+<b>The monthly job did precisely the inverse of its job.</b>
+<code>eval_corpus.py --refresh</code> walked every row of whatever file it was handed, so it moved
+the 20 required pins &mdash; which <code>build_scale_corpus.py</code> owns and copies verbatim
+&mdash; and, because <code>relock_text</code> skipped any row with fewer than two fields, pinned
+<b>zero</b> of the 165 bare distractors. The tier that exists to measure a pinned 20,000-entry
+corpus was floating on upstream HEAD for <b>89% of its rows</b>.
+
+<b>It also could never merge.</b> The same workflow runs
+<code>build_scale_corpus.py --check</code> one step <em>before</em> the refresh; the refresh then
+broke exactly that check. Every PR it opened was unmergeable by construction, and the diff read as
+ordinary pin movement &mdash; 20 SHAs and 20 counts, which is what a healthy re-pin looks like.
+PR #536 is the specimen.
+
+<b>Two mechanisms, both about a fallback being consulted too early.</b> The pins moved because
+nothing said the required rows belong to another file. The distractors stayed bare because
+<code>relock_text</code> gated on <em>&ldquo;does this row already have a pin&rdquo;</em> when the
+durable rule is <em>&ldquo;never write a count without a SHA&rdquo;</em> &mdash;
+<code>--relock</code> re-measures the same tree and has no SHA to offer, but <code>--refresh</code>
+has just resolved upstream HEAD and can close the row. A third, quieter symptom fell out of the
+first: the column width is the longest name being rewritten, and the distractor names are longer
+than anything in <code>taps.txt</code>, so every required row was re-columned even where its pin had
+not changed.
+
+<b>Freezing is by membership, not identity.</b> <code>--refresh</code> on <code>taps.txt</code>
+itself is exactly the job that <em>may</em> move those pins, so it freezes nothing. Freezing on
+&ldquo;is this a required repo&rdquo; would have made the required corpus permanently unrefreshable
+&mdash; the same bug pointed the other way, and far quieter.
+
+<b>Verified against the real file, not argued.</b> Simulating a refresh that resolved a new SHA for
+all 185 rows: the 20 required rows come back byte-identical, all 185 end up pinned, and
+<code>--check</code> passes &mdash; so the PR is mergeable. The workflow now runs that check
+<b>after</b> the refresh as well as before, which is the guard that was missing.</p></details>
+        <div class="meta">
+          <span class="m">Complexity <b>S</b></span>
+          <span class="m hi">Impact <b>High</b></span>
+          <span class="m">Wow <b class="wow">★★★★★</b></span>
+          <span class="m">185 rows, 20 pinned — the monthly job moved exactly the 20 it did not own and pinned none of the 165 it existed to pin</span>
+        </div>
+      </article>
       <!-- /ROADMAP:cards -->
 
     </div>

{boost_skill_cli-1.2.9 → boost_skill_cli-1.2.10}/scripts/eval_corpus.py

@@ -77,7 +77,7 @@ import argparse
 import re
 import subprocess
 import sys
-from collections.abc import Sequence
+from collections.abc import Collection, Sequence
 from pathlib import Path
 
 ROOT = Path(__file__).resolve().parent.parent
@@ -346,31 +346,50 @@ def _report_failures(failures: Sequence[CorpusError], total_rows: int) -> int:
 
 
 def relock_text(text: str, counts: dict[str, int],
-                shas: dict[str, str] | None = None) -> str:
+                shas: dict[str, str] | None = None,
+                frozen: Collection[str] | None = None) -> str:
     """Rewrite each pinned row's entry count — and its SHA when ``shas`` says so.
 
     A whole-file rewrite would lose the header, which is where the reasoning
-    lives; this touches only rows it has a new count for. A row with no SHA is
-    left alone — a count beside an unpinned repo describes a tree free to change
-    underneath it, so it must never be written.
+    lives; this touches only rows it has a new count for. A count is never
+    written without a SHA — beside an unpinned repo it would describe a tree
+    free to change underneath it — but the SHA may come from ``shas`` rather
+    than from the row, which is how ``--refresh`` closes a row that arrived
+    bare. Gating on "the row already has a pin" instead left the scale corpus
+    at 165 bare rows to 20 pinned, refreshing only the 20 it did not own.
 
     ``shas`` is what makes ``--refresh`` a rewrite of the pins rather than of
     the counts alone. It is optional because the two operations are genuinely
     different: ``--relock`` re-measures the *same* trees (the scanner changed),
     ``--refresh`` moves to *new* trees (the world changed).
+
+    ``frozen`` names repos whose rows this file does not own, and they are
+    emitted byte for byte — not re-pinned, and *not re-columned*. Both halves
+    matter: the width is the longest name being rewritten, so refreshing
+    ``taps-scale.txt`` (whose distractor names are longer than anything in
+    ``taps.txt``) moved every required row even where its pin had not changed.
     """
     shas = shas or {}
-    width = max([len(r) for r in counts] or [1])
+    frozen = frozen or ()
+    width = max([len(r) for r in counts if r not in frozen] or [1])
     out = []
     for raw in text.splitlines():
         line = raw.strip()
         parts = line.split()
-        if not line or line.startswith("#") or len(parts) < 2 \
-                or parts[0] not in counts or not _SHA.fullmatch(parts[1]):
+        if not line or line.startswith("#") or not parts \
+                or parts[0] not in counts or parts[0] in frozen:
             out.append(raw)
             continue
         repo = parts[0]
-        sha = shas.get(repo, parts[1])
+        # The row's own pin is only a fallback, and only when well formed: a
+        # malformed one is not a value to carry forward, it is a row nothing
+        # may claim to have measured.
+        committed = (parts[1] if len(parts) > 1
+                     and _SHA.fullmatch(parts[1]) else None)
+        sha = shas.get(repo) or committed
+        if sha is None:
+            out.append(raw)
+            continue
         if not _SHA.fullmatch(sha):
             raise SystemExit(
                 "refusing to write %s: %r is not a 40-character commit SHA"
@@ -379,6 +398,27 @@ def relock_text(text: str, counts: dict[str, int],
     return "\n".join(out) + ("\n" if text.endswith("\n") else "")
 
 
+def frozen_rows(taps: Path, required: Path = DEFAULT_TAPS) -> set[str]:
+    """Repos in ``taps`` whose pins belong to ``required``, not to ``taps``.
+
+    The scale corpus is the required corpus PLUS distractors, and
+    ``build_scale_corpus.render`` copies the required block in verbatim so both
+    tiers start from the same trees. The golden floors were calibrated against
+    those trees; a scale tier measuring a *different* snapshot of the same
+    repos reports a difference that is not scale.
+
+    Membership, not identity, decides. ``--refresh`` on ``taps.txt`` itself is
+    exactly the job that MAY move those pins, so it must freeze nothing —
+    freezing on "is this a required repo" rather than "does another file own
+    this row" would make the required corpus permanently unrefreshable, which
+    is the same bug pointed the other way and far quieter.
+    """
+    if taps.resolve() == required.resolve() or not required.exists():
+        return set()
+    return {repo for repo, _sha, _n in
+            parse_taps(required.read_text(encoding="utf-8"))}
+
+
 def refresh_summary(rows: Sequence[Row], shas: dict[str, str],
                     counts: dict[str, int]) -> str:
     """A Markdown table of what a refresh moved, for the pull request body.
@@ -465,10 +505,20 @@ def _refresh(taps: Path, summary_path: str | None = None) -> int:
     from boost_cli.core import catalog, gitutil, registry  # deferred: path shim
 
     rows = parse_taps(taps.read_text(encoding="utf-8"))
+    # Rows another file owns are neither measured nor written — skipping the
+    # measurement is not just an optimisation here, it is what keeps the two
+    # tiers pinned to one set of trees. It also saves re-cloning the required
+    # corpus, whose largest member is 6,309 entries.
+    frozen = frozen_rows(taps)
+    failures: list[CorpusError] = []
     shas: dict[str, str] = {}
     counts: dict[str, int] = {}
-    failures: list[CorpusError] = []
+    if frozen:
+        print("  %d rows are owned by %s and are left alone"
+              % (len(frozen & {r for r, _s, _n in rows}), DEFAULT_TAPS.name))
     for repo, old_sha, _n in rows:
+        if repo in frozen:
+            continue
         try:
             try:
                 tap = registry.get(repo)
@@ -499,8 +549,10 @@ def _refresh(taps: Path, summary_path: str | None = None) -> int:
                  "" if sha == old_sha else "   MOVED"))
     if failures:
         return _report_failures(failures, len(rows))
-    taps.write_text(relock_text(taps.read_text(encoding="utf-8"), counts, shas),
-                    encoding="utf-8")
+    taps.write_text(
+        relock_text(taps.read_text(encoding="utf-8"), counts, shas,
+                    frozen=frozen),
+        encoding="utf-8")
     summary = refresh_summary(rows, shas, counts)
     print("\n" + summary)
     if summary_path:

{boost_skill_cli-1.2.9 → boost_skill_cli-1.2.10}/tests/unit/test_eval_corpus.py

@@ -543,3 +543,156 @@ class TestTheSentinelIsKeyedOnTheTapList:
         out = self._wrapper_run(tmp_path, "owner/repo %s 2\n" % ("a" * 40), calls)
         assert "skipping" not in out
         assert calls.read_text(encoding="utf-8").count("ensure") == 2
+
+
+class TestRefreshingTheScaleCorpusLeavesTheRequiredRowsAlone:
+    """The required rows in `taps-scale.txt` belong to `taps.txt`.
+
+    `build_scale_corpus.render` copies them in verbatim — pin and count
+    included — because both tiers must start from the same trees: the golden
+    floors were calibrated against the required corpus, and a scale tier
+    measuring a *different* snapshot of the same repos reports a difference
+    that is not scale. The generator's own section header says "verbatim".
+
+    Nothing enforced it. `--refresh` walked every row of whatever file it was
+    given, so the monthly scale job moved the required pins too, and the two
+    tiers silently decoupled. It also re-columned the whole file (the width is
+    the longest name in the file, and the distractor names are longer), so
+    every required row changed even where the pin did not.
+
+    The PR it opened could never merge: `build_scale_corpus.py --check` — which
+    the same workflow runs one step earlier — calls the result stale, forever.
+    """
+
+    def _scale_file(self, tmp_path):
+        required = ("anthropics/skills     %s    18\n" % ("a" * 40)
+                    + "minio/skills          %s     3\n" % ("b" * 40))
+        scale = tmp_path / "taps-scale.txt"
+        scale.write_text(
+            "# --- the required corpus, verbatim ---\n"
+            + required
+            + "\n# --- distractors ---\n"
+            + "some-very-long-owner/some-very-long-repo-name\n",
+            encoding="utf-8")
+        (tmp_path / "taps.txt").write_text(required, encoding="utf-8")
+        return scale, required
+
+    def test_a_required_row_is_not_re_pinned(self, tmp_path):
+        scale, required = self._scale_file(tmp_path)
+        frozen = _load().frozen_rows(scale, tmp_path / "taps.txt")
+        assert frozen == {"anthropics/skills", "minio/skills"}
+        # A refresh that measured new trees for them must still not write them.
+        out = _load().relock_text(
+            scale.read_text(encoding="utf-8"),
+            {"anthropics/skills": 20, "minio/skills": 9},
+            {"anthropics/skills": "c" * 40, "minio/skills": "d" * 40},
+            frozen=frozen)
+        for row in required.splitlines():
+            assert row in out, "required row was rewritten: %r" % row
+
+    def test_a_frozen_row_keeps_its_column_width(self, tmp_path):
+        """The subtler half: even an unchanged pin was being re-columned.
+
+        `relock_text` pads to the longest name it is rewriting, and the scale
+        file's distractors are longer than anything in `taps.txt` — so the
+        required block came back at a different width and stopped matching the
+        generator byte for byte.
+        """
+        scale, _required = self._scale_file(tmp_path)
+        out = _load().relock_text(
+            scale.read_text(encoding="utf-8"),
+            {"anthropics/skills": 18,
+             "some-very-long-owner/some-very-long-repo-name": 7},
+            {"some-very-long-owner/some-very-long-repo-name": "e" * 40},
+            frozen={"anthropics/skills"})
+        assert "anthropics/skills     %s    18" % ("a" * 40) in out
+        assert "some-very-long-owner/some-very-long-repo-name %s" % ("e" * 40) in out
+
+    def test_distractors_are_still_refreshed(self, tmp_path):
+        # Freezing must not turn the monthly job into a no-op.
+        scale, _required = self._scale_file(tmp_path)
+        out = _load().relock_text(
+            scale.read_text(encoding="utf-8"),
+            {"some-very-long-owner/some-very-long-repo-name": 7},
+            {"some-very-long-owner/some-very-long-repo-name": "e" * 40},
+            frozen={"anthropics/skills", "minio/skills"})
+        assert "some-very-long-owner/some-very-long-repo-name %s     7" % ("e" * 40) in out
+
+    def test_refreshing_the_required_corpus_itself_freezes_nothing(self, tmp_path):
+        """`--refresh` on taps.txt is exactly the job that MAY move those pins.
+
+        Freezing on identity rather than on membership would make the required
+        corpus permanently unrefreshable — the opposite failure, and a quieter
+        one.
+        """
+        _scale, required = self._scale_file(tmp_path)
+        taps = tmp_path / "taps.txt"
+        assert _load().frozen_rows(taps, taps) == set()
+        out = _load().relock_text(required, {"anthropics/skills": 20},
+                             {"anthropics/skills": "c" * 40},
+                             frozen=_load().frozen_rows(taps, taps))
+        assert "c" * 40 in out
+
+    def test_a_missing_required_file_freezes_nothing(self, tmp_path):
+        # A refresh must degrade to the old behaviour, never crash.
+        scale, _required = self._scale_file(tmp_path)
+        assert _load().frozen_rows(scale, tmp_path / "absent.txt") == set()
+
+
+class TestRefreshPinsARowThatArrivedBare:
+    """`--refresh` resolves a new SHA, so it can pin a row that has none.
+
+    `relock_text` skipped any row with fewer than two fields, on a rule that is
+    right for `--relock` and wrong for `--refresh`. The two differ in exactly
+    the way that matters: `--relock` re-measures the *same* tree and has no SHA
+    to offer, so writing a count beside an unpinned repo would describe a tree
+    free to change underneath it. `--refresh` has just resolved upstream HEAD,
+    so it has both halves and the row can be closed.
+
+    Measured cost of the old rule: the shipped scale corpus is 165 bare rows
+    and 20 pinned ones, and the monthly job rewrote exactly the 20 — the
+    required block it must not touch — while pinning none of the 165 it exists
+    to pin. The tier that is supposed to measure a pinned 20,000-entry corpus
+    was floating on upstream HEAD for 89% of its rows.
+
+    The durable rule is about the SHA, not about the row: never write a count
+    without one.
+    """
+
+    def test_a_bare_row_gains_a_pin_when_a_sha_was_resolved(self):
+        m = _load()
+        out = m.relock_text("a/b\n", {"a/b": 7}, {"a/b": "f" * 40})
+        assert out == "a/b %s     7\n" % ("f" * 40)
+
+    def test_a_bare_row_is_left_alone_when_no_sha_was_resolved(self):
+        """The property the old rule was protecting, kept.
+
+        This is `--relock`: the scanner changed, the trees did not, and there
+        is no SHA on offer. A count here would claim a measurement of a tree
+        nobody pinned.
+        """
+        m = _load()
+        assert m.relock_text("a/b\n", {"a/b": 7}) == "a/b\n"
+
+    def test_a_bare_row_with_no_count_is_left_alone(self):
+        m = _load()
+        assert m.relock_text("a/b\n", {}, {"a/b": "f" * 40}) == "a/b\n"
+
+    def test_a_malformed_pin_is_still_refused(self):
+        # Not a SHA and nothing resolved for it: writing would launder a typo
+        # into a row that reads as measured.
+        m = _load()
+        assert m.relock_text("a/b nope 3\n", {"a/b": 7}) == "a/b nope 3\n"
+
+    def test_a_malformed_pin_is_replaced_when_a_sha_was_resolved(self):
+        m = _load()
+        out = m.relock_text("a/b nope 3\n", {"a/b": 7}, {"a/b": "f" * 40})
+        assert out == "a/b %s     7\n" % ("f" * 40)
+
+    def test_the_width_covers_rows_that_were_bare(self):
+        # A newly-pinned long name must not be excluded from the column maths.
+        m = _load()
+        out = m.relock_text("short/x\nmuch-longer-owner/repo\n",
+                            {"short/x": 1, "much-longer-owner/repo": 2},
+                            {"short/x": "a" * 40, "much-longer-owner/repo": "b" * 40})
+        assert "short/x                %s     1" % ("a" * 40) in out

{boost_skill_cli-1.2.9 → boost_skill_cli-1.2.10}/tests/unit/test_scale_corpus.py

@@ -230,3 +230,37 @@ class TestPinsSurviveRegeneration:
         refreshed = first.replace("\ns/one\n", "\ns/one %s 99\n" % ("d" * 40))
         again = m.render(required, picked, 30, m.existing_pins(refreshed))
         assert again == refreshed
+
+
+class TestTheScheduledJobCannotOpenAnUnmergeablePR:
+    """`--check` must run AFTER the refresh, not only before it.
+
+    Running it first proves the committed file is fresh; it says nothing about
+    what the refresh then wrote. For months the job checked, refreshed, and
+    opened a PR that the very same check would have rejected — because
+    `--refresh` re-pinned and re-columned the required block, which the
+    generator copies verbatim from `taps.txt`. The PR was unmergeable by
+    construction and the diff read as ordinary pin movement.
+    """
+
+    _WORKFLOW = _ROOT / ".github" / "workflows" / "eval-scale.yml"
+
+    @pytest.mark.skipif(not _WORKFLOW.exists(), reason="workflow not reachable")
+    def test_the_check_runs_after_the_refresh(self):
+        text = self._WORKFLOW.read_text(encoding="utf-8")
+        check = "build_scale_corpus.py --check"
+        refresh = "eval_corpus.py --refresh"
+        assert text.count(check) >= 2, (
+            "only one --check: a refresh that breaks the selection would still "
+            "open a PR")
+        assert text.index(refresh) < text.rindex(check), (
+            "the last --check must follow the refresh")
+
+    @pytest.mark.skipif(not _WORKFLOW.exists(), reason="workflow not reachable")
+    def test_the_check_is_not_allowed_to_fail_quietly(self):
+        # `score it` is continue-on-error by design; the selection check is not.
+        text = self._WORKFLOW.read_text(encoding="utf-8")
+        tail = text[text.rindex("build_scale_corpus.py --check"):]
+        head = text[:text.rindex("build_scale_corpus.py --check")]
+        step = head[head.rindex("      - name:"):] + tail.split("      - name:")[0]
+        assert "continue-on-error" not in step