boost-skill-cli 1.0.359__tar.gz → 1.0.360__tar.gz

{boost_skill_cli-1.0.359 → boost_skill_cli-1.0.360}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: boost-skill-cli
-Version: 1.0.359
+Version: 1.0.360
 Summary: boost — Homebrew for AI coding skills
 Author: Jonathan Reyes
 License: GNU GENERAL PUBLIC LICENSE

{boost_skill_cli-1.0.359 → boost_skill_cli-1.0.360}/boost_cli/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '1.0.359'
-__version_tuple__ = version_tuple = (1, 0, 359)
+__version__ = version = '1.0.360'
+__version_tuple__ = version_tuple = (1, 0, 360)
 
 __commit_id__ = commit_id = None

{boost_skill_cli-1.0.359 → boost_skill_cli-1.0.360}/boost_skill_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: boost-skill-cli
-Version: 1.0.359
+Version: 1.0.360
 Summary: boost — Homebrew for AI coding skills
 Author: Jonathan Reyes
 License: GNU GENERAL PUBLIC LICENSE

{boost_skill_cli-1.0.359 → boost_skill_cli-1.0.360}/docs/roadmap/items/golden-set-grades-by-name-not-by-skill.md

@@ -7,10 +7,10 @@ category: Eval · Correctness
 complexity: M
 impact: High
 wow: 4
-note: the mechanism ships; which of 59 code-reviewers each query means is still a human call
+note: 28 of 50 rows pinned mechanically with zero change to any number; 22 are genuine judgment calls
 order: 81
 owner: loop/golden-name-grading
-pr:
+pr: 412
 title: The golden set grades by name, and 35 of 53 names are ambiguous
 ---
 <b>The eval scores a hit when the top result carries the right <em>name</em>. Most of those names do
@@ -84,3 +84,24 @@ statement about intent. Guessing it would bake one opinion into the number the p
 invisibly. The harness is ready for those decisions one row at a time; each added exemplar tightens
 the metric and none of them destabilise it.
 
+<b>Progress: 28 of the 50 rows are now pinned, and pinning them changed nothing.</b> Measured over
+the SHA-pinned corpus, 28 rows have the property that every name in their <code>relevant</code> list
+resolves to exactly <b>one body</b> &mdash; so the exemplar is a lookup, not a judgment, and grading
+by content class must return the same verdict as grading by name. It does: the natural-language set
+scores <b>0.350 / 0.160 / 0.245 / 0.259</b> before and after, identical to three decimal places.
+That equality is the point of shipping them &mdash; the rows are now explicit about which skill they
+mean, at zero cost to comparability.
+
+<b>The remaining 22 are the real content of this card.</b> <code>code-reviewer</code> is 13 distinct
+skills in this corpus, <code>update-docs</code> 10, <code>commit</code> 4. There is no shortcut
+available: their descriptions share a median similarity of about <b>0.15</b>, so these are genuine
+forks rather than one skill re-published, and no rule separates them without someone saying what the
+question meant. The menu is generated rather than written down, because the candidate set is a fact
+about the corpus that is tapped:
+
+<code>python3 scripts/eval_retrieval.py --golden tests/eval/golden-natural.jsonl --worksheet</code>
+
+<b>What unblocked this.</b> Not the judgment calls &mdash; [[eval-deduped-ranked-lists-by-name]].
+A half-migrated set was averaging two different rank conventions, because name-graded rows collapsed
+homonyms into one rank slot while exemplar-graded rows gave every distractor mirror its own. Until
+both used one convention, migrating rows one at a time produced a number that meant nothing.

{boost_skill_cli-1.0.359 → boost_skill_cli-1.0.360}/docs/roadmap.html

@@ -2412,12 +2412,34 @@ weaker gate that still reports a number, which is the failure this card exists t
 <code>code-reviewer</code>s a query about reviewing a diff for security problems refers to is a
 statement about intent. Guessing it would bake one opinion into the number the project publishes,
 invisibly. The harness is ready for those decisions one row at a time; each added exemplar tightens
-the metric and none of them destabilise it.</p>
+the metric and none of them destabilise it.
+
+<b>Progress: 28 of the 50 rows are now pinned, and pinning them changed nothing.</b> Measured over
+the SHA-pinned corpus, 28 rows have the property that every name in their <code>relevant</code> list
+resolves to exactly <b>one body</b> &mdash; so the exemplar is a lookup, not a judgment, and grading
+by content class must return the same verdict as grading by name. It does: the natural-language set
+scores <b>0.350 / 0.160 / 0.245 / 0.259</b> before and after, identical to three decimal places.
+That equality is the point of shipping them &mdash; the rows are now explicit about which skill they
+mean, at zero cost to comparability.
+
+<b>The remaining 22 are the real content of this card.</b> <code>code-reviewer</code> is 13 distinct
+skills in this corpus, <code>update-docs</code> 10, <code>commit</code> 4. There is no shortcut
+available: their descriptions share a median similarity of about <b>0.15</b>, so these are genuine
+forks rather than one skill re-published, and no rule separates them without someone saying what the
+question meant. The menu is generated rather than written down, because the candidate set is a fact
+about the corpus that is tapped:
+
+<code>python3 scripts/eval_retrieval.py --golden tests/eval/golden-natural.jsonl --worksheet</code>
+
+<b>What unblocked this.</b> Not the judgment calls &mdash; [[eval-deduped-ranked-lists-by-name]].
+A half-migrated set was averaging two different rank conventions, because name-graded rows collapsed
+homonyms into one rank slot while exemplar-graded rows gave every distractor mirror its own. Until
+both used one convention, migrating rows one at a time produced a number that meant nothing.</p>
         <div class="meta">
           <span class="m">Complexity <b>M</b></span>
           <span class="m hi">Impact <b>High</b></span>
           <span class="m">Wow <b class="wow">★★★★</b></span>
-          <span class="m">the mechanism ships; which of 59 code-reviewers each query means is still a human call</span>
+          <span class="m">28 of 50 rows pinned mechanically with zero change to any number; 22 are genuine judgment calls</span>
         </div>
       </article>
 

{boost_skill_cli-1.0.359 → boost_skill_cli-1.0.360}/scripts/eval_retrieval.py

@@ -213,6 +213,46 @@ def relevant_keys(row: dict) -> set:
     return {"cls:%s" % sorted(classes)[0]}
 
 
+def exemplar_worksheet(rows: List[dict], entries: List[dict],
+                       hashes: dict) -> List[dict]:
+    """Rows still graded by name whose name resolves to several bodies.
+
+    Pinning those is a judgment about what the question meant, not a lookup, so
+    this hands over the menu rather than guessing: for each undecided row, every
+    distinct body a relevant name resolves to, with its description.
+
+    Generated rather than committed as a comment in the golden file, because the
+    candidate list is a fact about the corpus that is currently tapped. Written
+    down, it would be wrong the first time a pin moves.
+    """
+    by_name: Dict[str, List[dict]] = {}
+    for entry in entries:
+        by_name.setdefault(str(entry.get("name", "")), []).append(entry)
+    sheet: List[dict] = []
+    for row in rows:
+        if row.get("class_hashes"):
+            continue                       # already decided
+        seen: Dict[str, dict] = {}
+        for name in row["relevant_set"]:
+            for entry in by_name.get(name, []):
+                digest = hashes.get((entry.get("tap", ""), entry.get("skill_md", "")))
+                if digest and digest not in seen:
+                    seen[digest] = entry
+        if len(seen) < 2:
+            continue                       # determined, or absent entirely
+        sheet.append({
+            "query": row["query"],
+            "candidates": [
+                {"spec": "%s%s%s" % (e.get("tap", ""), _EXEMPLAR_SEP,
+                                     e.get("skill_md", "")),
+                 "description": (e.get("description") or "").strip().split("\n")[0]}
+                for e in sorted(seen.values(),
+                                key=lambda x: (x.get("tap", ""), x.get("skill_md", "")))
+            ],
+        })
+    return sheet
+
+
 def dedupe_keys(keys):
     """Collapse to the first (best-ranked) occurrence of each key."""
     seen: set = set()
@@ -603,6 +643,9 @@ def main(argv: Optional[List[str]] = None) -> int:
                     help="Tier 1b: ranx significance test between engines "
                          "(opt-in [eval] extra; degrades if ranx absent)")
     ap.add_argument("--json", action="store_true", help="machine-readable JSON")
+    ap.add_argument("--worksheet", action="store_true",
+                    help="list the golden rows still graded by name whose name "
+                         "resolves to several bodies, with the candidates")
     args = ap.parse_args(argv)
 
     floors = parse_floors(args.floor)          # fail fast on a bad --floor
@@ -616,6 +659,22 @@ def main(argv: Optional[List[str]] = None) -> int:
             print("  indexed %d entries -> %d chunks across %d taps"
                   % (stats["entries"], stats["docs"], stats["taps"]))
 
+    if args.worksheet:
+        sheet = exemplar_worksheet(rows, catalog.all_entries(), rag.content_hashes())
+        if args.json:
+            print(json.dumps(sheet, indent=2))
+            return 0
+        print("%d of %d rows still graded by name resolve to several bodies.\n"
+              % (len(sheet), len(rows)))
+        for case in sheet:
+            print("  %s" % case["query"])
+            for cand in case["candidates"]:
+                print("      %s" % cand["spec"])
+                if cand["description"]:
+                    print("          %s" % cand["description"][:96])
+            print()
+        return 0
+
     if args.rerank:
         return run_rerank_lift(rows, args.k, args.json)
 

{boost_skill_cli-1.0.359 → boost_skill_cli-1.0.360}/tests/eval/golden-natural.jsonl

@@ -31,25 +31,45 @@
 # than flooring one. Adding it to the gate would also floor a number nobody has
 # argued for yet.
 #
-# One JSON object per line: {query, relevant[], kind, note}
-{"query": "my writing is too wordy and people skim past the important parts", "relevant": ["write-concisely"], "kind": "skill", "note": "clarity, no shared vocabulary with the name"}
-{"query": "an error surfaces deep in the call stack and I cannot tell what originally triggered it", "relevant": ["root-cause-tracing"], "kind": "skill", "note": "tracing back from a symptom"}
-{"query": "I keep patching symptoms instead of whatever is actually wrong underneath", "relevant": ["why", "root-cause-tracing"], "kind": "skill", "note": "five-whys intent"}
+# GRADING. A row may pin an `exemplar` — "tap::skill_md", the entry the query was
+# written about — and is then graded on that entry's CONTENT CLASS: byte-identical
+# mirrors from other registries still count, a different skill sharing the name
+# does not. 28 of these 50 rows carry one. They were not chosen: over the pinned
+# corpus each of their relevant names resolves to exactly ONE body, so the pin is
+# a lookup rather than a judgment, and the pinned taps.txt is what makes that
+# reproducible.
+#
+# The other 22 are genuinely undecided. `code-reviewer` is 13 different skills
+# here, `update-docs` 10, `commit` 4, and their descriptions share a median
+# similarity of ~0.15 — there is no "same skill re-published" shortcut. Deciding
+# which one a question meant is a statement about intent, and guessing it would
+# bake one opinion into a published number. To see the menu:
+#
+#   python3 scripts/eval_retrieval.py --golden tests/eval/golden-natural.jsonl --worksheet
+#
+# which prints every candidate body and its description. It is generated rather
+# than listed here on purpose: the candidate set is a fact about the corpus that
+# is tapped, so written down it would be wrong the first time a pin moves.
+#
+# One JSON object per line: {query, relevant[], exemplar?, kind, note}
+{"query": "my writing is too wordy and people skim past the important parts", "relevant": ["write-concisely"], "exemplar": "NeoLabHQ/context-engineering-kit::plugins/docs/skills/write-concisely/SKILL.md", "kind": "skill", "note": "clarity, no shared vocabulary with the name"}
+{"query": "an error surfaces deep in the call stack and I cannot tell what originally triggered it", "relevant": ["root-cause-tracing"], "exemplar": "NeoLabHQ/context-engineering-kit::plugins/kaizen/skills/root-cause-tracing/SKILL.md", "kind": "skill", "note": "tracing back from a symptom"}
+{"query": "I keep patching symptoms instead of whatever is actually wrong underneath", "relevant": ["why", "root-cause-tracing"], "exemplar": ["NeoLabHQ/context-engineering-kit::plugins/kaizen/skills/why/SKILL.md", "NeoLabHQ/context-engineering-kit::plugins/kaizen/skills/root-cause-tracing/SKILL.md"], "kind": "skill", "note": "five-whys intent"}
 {"query": "what convention should I follow for the messages I write when recording changes in git", "relevant": ["commit"], "kind": "skill", "note": "conventional commits"}
-{"query": "I keep stashing half-finished work every time I have to switch branches", "relevant": ["git-worktrees"], "kind": "skill", "note": "worktrees without naming them"}
-{"query": "I want to attach review status to a commit without rewriting any history", "relevant": ["git-notes"], "kind": "skill", "note": "metadata on commits"}
+{"query": "I keep stashing half-finished work every time I have to switch branches", "relevant": ["git-worktrees"], "exemplar": "NeoLabHQ/context-engineering-kit::plugins/git/skills/git-worktrees/SKILL.md", "kind": "skill", "note": "worktrees without naming them"}
+{"query": "I want to attach review status to a commit without rewriting any history", "relevant": ["git-notes"], "exemplar": "NeoLabHQ/context-engineering-kit::plugins/git/skills/git-notes/SKILL.md", "kind": "skill", "note": "metadata on commits"}
 {"query": "check what I changed for injection flaws and leaked credentials before it goes in", "relevant": ["security-auditor"], "kind": "workflow", "note": "vulnerability review"}
-{"query": "am I actually exercising the risky paths or only the easy ones", "relevant": ["test-coverage-reviewer"], "kind": "workflow", "note": "coverage quality"}
+{"query": "am I actually exercising the risky paths or only the easy ones", "relevant": ["test-coverage-reviewer"], "exemplar": "NeoLabHQ/context-engineering-kit::plugins/review/agents/test-coverage-reviewer.md", "kind": "workflow", "note": "coverage quality"}
 {"query": "read through my diff and tell me what is broken in it", "relevant": ["bug-hunter"], "kind": "workflow", "note": "defect finding"}
 {"query": "does this change follow the conventions the rest of the project uses", "relevant": ["code-reviewer"], "kind": "workflow", "note": "guideline adherence"}
-{"query": "I need a single page covering what went wrong, why it happened and what we will do", "relevant": ["analyse-problem"], "kind": "skill", "note": "A3 report"}
-{"query": "help me lay out everything that might be contributing, grouped into categories", "relevant": ["cause-and-effect"], "kind": "skill", "note": "fishbone"}
-{"query": "I want to run small controlled experiments and keep iterating on the result", "relevant": ["plan-do-check-act"], "kind": "skill", "note": "PDCA"}
+{"query": "I need a single page covering what went wrong, why it happened and what we will do", "relevant": ["analyse-problem"], "exemplar": "NeoLabHQ/context-engineering-kit::plugins/kaizen/skills/analyse-problem/SKILL.md", "kind": "skill", "note": "A3 report"}
+{"query": "help me lay out everything that might be contributing, grouped into categories", "relevant": ["cause-and-effect"], "exemplar": "NeoLabHQ/context-engineering-kit::plugins/kaizen/skills/cause-and-effect/SKILL.md", "kind": "skill", "note": "fishbone"}
+{"query": "I want to run small controlled experiments and keep iterating on the result", "relevant": ["plan-do-check-act"], "exemplar": "NeoLabHQ/context-engineering-kit::plugins/kaizen/skills/plan-do-check-act/SKILL.md", "kind": "skill", "note": "PDCA"}
 {"query": "drive my locally running site in a real browser and confirm it behaves", "relevant": ["webapp-testing"], "kind": "skill", "note": "playwright without the word"}
 {"query": "get the text out of a scanned document so I can search it", "relevant": ["pdf"], "kind": "skill", "note": "extraction"}
-{"query": "I need to edit a Word file programmatically", "relevant": ["docx"], "kind": "skill", "note": "office format"}
-{"query": "read numbers out of a spreadsheet and write some back", "relevant": ["xlsx"], "kind": "skill", "note": "office format"}
-{"query": "put together a slide deck for a presentation", "relevant": ["pptx"], "kind": "skill", "note": "office format"}
+{"query": "I need to edit a Word file programmatically", "relevant": ["docx"], "exemplar": "anthropics/skills::skills/docx/SKILL.md", "kind": "skill", "note": "office format"}
+{"query": "read numbers out of a spreadsheet and write some back", "relevant": ["xlsx"], "exemplar": "anthropics/skills::skills/xlsx/SKILL.md", "kind": "skill", "note": "office format"}
+{"query": "put together a slide deck for a presentation", "relevant": ["pptx"], "exemplar": "anthropics/skills::skills/pptx/SKILL.md", "kind": "skill", "note": "office format"}
 {"query": "make a short looping animation small enough to post in chat", "relevant": ["slack-gif-creator"], "kind": "skill", "note": "gif constraints"}
 {"query": "make this match our company colours and typefaces", "relevant": ["brand-guidelines"], "kind": "skill", "note": "brand application"}
 {"query": "the interface looks generic and I want it to feel deliberate", "relevant": ["frontend-design"], "kind": "skill", "note": "visual design intent"}
@@ -58,27 +78,27 @@
 {"query": "I am running out of room in the window and important details get dropped", "relevant": ["context-engineering"], "kind": "skill", "note": "context limits"}
 {"query": "I want to package a repeatable procedure so it can be reused later", "relevant": ["skill-creator", "create-skill"], "kind": "skill", "note": "duplicate intent in corpus"}
 {"query": "how do I tell whether my changes to the instructions actually helped", "relevant": ["agent-evaluation"], "kind": "skill", "note": "measuring prompt changes"}
-{"query": "run something automatically every time before code is committed", "relevant": ["create-hook"], "kind": "skill", "note": "git hooks"}
-{"query": "turn this ticket into a proper technical specification", "relevant": ["analyze-issue"], "kind": "skill", "note": "issue to spec"}
+{"query": "run something automatically every time before code is committed", "relevant": ["create-hook"], "exemplar": "NeoLabHQ/context-engineering-kit::plugins/customaize-agent/skills/create-hook/SKILL.md", "kind": "skill", "note": "git hooks"}
+{"query": "turn this ticket into a proper technical specification", "relevant": ["analyze-issue"], "exemplar": "NeoLabHQ/context-engineering-kit::plugins/git/skills/analyze-issue/SKILL.md", "kind": "skill", "note": "issue to spec"}
 {"query": "open a change request with the right template filled in", "relevant": ["create-pr"], "kind": "skill", "note": "PR creation"}
-{"query": "gather all the unresolved feedback on my change into a task list", "relevant": ["load-pr-comments"], "kind": "skill", "note": "review comments"}
+{"query": "gather all the unresolved feedback on my change into a task list", "relevant": ["load-pr-comments"], "exemplar": "NeoLabHQ/context-engineering-kit::plugins/git/skills/load-pr-comments/SKILL.md", "kind": "skill", "note": "review comments"}
 {"query": "the written guides no longer match what the code does", "relevant": ["update-docs"], "kind": "skill", "note": "stale documentation"}
 {"query": "walk me through writing a guide together step by step", "relevant": ["doc-coauthoring"], "kind": "skill", "note": "structured authoring"}
-{"query": "this needs careful multi-step logic rather than a quick guess", "relevant": ["thought-based-reasoning"], "kind": "skill", "note": "reasoning depth"}
-{"query": "I want several different perspectives to argue about my design before I commit to it", "relevant": ["critique"], "kind": "skill", "note": "multi-judge debate"}
-{"query": "look back over what you just produced and improve on it", "relevant": ["reflect"], "kind": "skill", "note": "self-refinement"}
-{"query": "keep this lesson around so the same mistake is not repeated", "relevant": ["memorize"], "kind": "skill", "note": "persisting insights"}
-{"query": "some of the decisions we recorded are probably out of date now", "relevant": ["decay"], "kind": "skill", "note": "evidence freshness"}
+{"query": "this needs careful multi-step logic rather than a quick guess", "relevant": ["thought-based-reasoning"], "exemplar": "NeoLabHQ/context-engineering-kit::plugins/customaize-agent/skills/thought-based-reasoning/SKILL.md", "kind": "skill", "note": "reasoning depth"}
+{"query": "I want several different perspectives to argue about my design before I commit to it", "relevant": ["critique"], "exemplar": "NeoLabHQ/context-engineering-kit::plugins/reflexion/skills/critique/SKILL.md", "kind": "skill", "note": "multi-judge debate"}
+{"query": "look back over what you just produced and improve on it", "relevant": ["reflect"], "exemplar": "NeoLabHQ/context-engineering-kit::plugins/reflexion/skills/reflect/SKILL.md", "kind": "skill", "note": "self-refinement"}
+{"query": "keep this lesson around so the same mistake is not repeated", "relevant": ["memorize"], "exemplar": "NeoLabHQ/context-engineering-kit::plugins/reflexion/skills/memorize/SKILL.md", "kind": "skill", "note": "persisting insights"}
+{"query": "some of the decisions we recorded are probably out of date now", "relevant": ["decay"], "exemplar": "NeoLabHQ/context-engineering-kit::plugins/fpf/skills/decay/SKILL.md", "kind": "skill", "note": "evidence freshness"}
 {"query": "generate visuals from code with a bit of controlled randomness", "relevant": ["algorithmic-art"], "kind": "skill", "note": "creative coding"}
 {"query": "produce a good looking printable document rather than a web page", "relevant": ["canvas-design"], "kind": "skill", "note": "print output"}
 {"query": "apply one consistent look across slides, documents and pages", "relevant": ["theme-factory"], "kind": "skill", "note": "theming"}
 {"query": "what does calling the model actually cost per token", "relevant": ["claude-api"], "kind": "skill", "note": "pricing reference"}
 {"query": "my containers keep restarting and I cannot work out why", "relevant": ["docker-expert"], "kind": "workflow", "note": "avoids the word docker"}
-{"query": "the layout falls apart on a narrow screen", "relevant": ["css-expert"], "kind": "workflow", "note": "responsive layout"}
-{"query": "I want automated tests that click through the app like a person would", "relevant": ["cypress-expert"], "kind": "workflow", "note": "e2e testing"}
-{"query": "our full-text queries have got slow as the index grew", "relevant": ["elasticsearch-expert"], "kind": "workflow", "note": "search performance"}
-{"query": "how should I choose partition and sort keys for this access pattern", "relevant": ["dynamodb-expert"], "kind": "workflow", "note": "key design"}
-{"query": "my shell script fails silently in the pipeline and I never notice", "relevant": ["bash-expert"], "kind": "workflow", "note": "defensive scripting"}
-{"query": "I need to configure a fleet of servers repeatably without doing it by hand", "relevant": ["ansible-expert"], "kind": "workflow", "note": "config management"}
-{"query": "keep database schema versions in step across environments", "relevant": ["flyway-expert"], "kind": "workflow", "note": "migrations"}
-{"query": "background jobs are piling up and not getting processed", "relevant": ["celery-expert", "bullmq-expert"], "kind": "workflow", "note": "two queue libraries, both defensible"}
+{"query": "the layout falls apart on a narrow screen", "relevant": ["css-expert"], "exemplar": "0xfurai/claude-code-subagents::agents/css-expert.md", "kind": "workflow", "note": "responsive layout"}
+{"query": "I want automated tests that click through the app like a person would", "relevant": ["cypress-expert"], "exemplar": "0xfurai/claude-code-subagents::agents/cypress-expert.md", "kind": "workflow", "note": "e2e testing"}
+{"query": "our full-text queries have got slow as the index grew", "relevant": ["elasticsearch-expert"], "exemplar": "0xfurai/claude-code-subagents::agents/elasticsearch-expert.md", "kind": "workflow", "note": "search performance"}
+{"query": "how should I choose partition and sort keys for this access pattern", "relevant": ["dynamodb-expert"], "exemplar": "0xfurai/claude-code-subagents::agents/dynamodb-expert.md", "kind": "workflow", "note": "key design"}
+{"query": "my shell script fails silently in the pipeline and I never notice", "relevant": ["bash-expert"], "exemplar": "0xfurai/claude-code-subagents::agents/bash-expert.md", "kind": "workflow", "note": "defensive scripting"}
+{"query": "I need to configure a fleet of servers repeatably without doing it by hand", "relevant": ["ansible-expert"], "exemplar": "0xfurai/claude-code-subagents::agents/ansible-expert.md", "kind": "workflow", "note": "config management"}
+{"query": "keep database schema versions in step across environments", "relevant": ["flyway-expert"], "exemplar": "0xfurai/claude-code-subagents::agents/flyway-expert.md", "kind": "workflow", "note": "migrations"}
+{"query": "background jobs are piling up and not getting processed", "relevant": ["celery-expert", "bullmq-expert"], "exemplar": ["0xfurai/claude-code-subagents::agents/celery-expert.md", "0xfurai/claude-code-subagents::agents/bullmq-expert.md"], "kind": "workflow", "note": "two queue libraries, both defensible"}

{boost_skill_cli-1.0.359 → boost_skill_cli-1.0.360}/tests/unit/test_eval_grading.py

@@ -28,6 +28,7 @@ exemplar-graded row and a name-graded row be averaged into one number.
 from __future__ import annotations
 
 import importlib.util
+import json
 from pathlib import Path
 
 import pytest
@@ -178,6 +179,50 @@ class TestIdentityIsTheBodyNotTheName:
         assert m.grade_key(row, one, HASHES) != m.grade_key(row, two, HASHES)
 
 
+class TestTheWorksheetShowsWhatIsLeftToDecide:
+    """Pinning the remaining rows is a judgment, so hand over a menu, not a task.
+
+    28 of the 50 natural-language rows resolved to exactly one body and were
+    pinned mechanically. The other 22 do not: `code-reviewer` alone is 13
+    different skills here, and their descriptions share a median similarity of
+    about 0.15, so there is no "same skill re-published" shortcut to take. The
+    menu is generated rather than written into the golden file as a comment, so
+    it keeps describing the corpus that is actually tapped instead of rotting.
+    """
+
+    def test_an_ambiguous_row_lists_every_candidate_body(self):
+        m = _load()
+        rows = [m.prepare_row({"query": "q", "relevant": ["code-reviewer"]}, HASHES)]
+        sheet = m.exemplar_worksheet(rows, [A, B, C, OTHER], HASHES)
+        assert len(sheet) == 1
+        specs = {c["spec"] for c in sheet[0]["candidates"]}
+        assert specs == {"owner/a::code-reviewer/SKILL.md",
+                         "owner/b::code-reviewer/SKILL.md"}
+
+    def test_mirrors_are_one_candidate_not_two(self):
+        # owner/c is byte-identical to owner/a; offering both would be offering
+        # the same decision twice.
+        m = _load()
+        rows = [m.prepare_row({"query": "q", "relevant": ["code-reviewer"]}, HASHES)]
+        assert m.exemplar_worksheet(rows, [A, C], HASHES) == []
+
+    def test_a_row_that_already_pins_an_exemplar_is_done(self):
+        m = _load()
+        rows = [m.prepare_row({"query": "q", "relevant": ["code-reviewer"],
+                               "exemplar": "owner/a::code-reviewer/SKILL.md"},
+                              HASHES)]
+        assert m.exemplar_worksheet(rows, [A, B, C], HASHES) == []
+
+    def test_the_shipped_set_leaves_exactly_the_rows_it_documents(self):
+        # Guards the split: if a future edit pins one of the 22 or unpins one of
+        # the 28, this is the line that says so.
+        path = _ROOT / "tests" / "eval" / "golden-natural.jsonl"
+        rows = [json.loads(ln) for ln in path.read_text(encoding="utf-8").splitlines()
+                if ln.strip() and not ln.startswith("#")]
+        assert len(rows) == 50
+        assert len([r for r in rows if r.get("exemplar")]) == 28
+
+
 class TestDedupeKeepsTheBestRank:
     def test_repeats_collapse_to_first_occurrence(self):
         m = _load()